From a67b5cb6b32c61588eefdb76099545fcfcd3fcc0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 7 Jan 2017 00:56:01 +0000 Subject: [PATCH 001/292] Avoid duplicate b-tree searches in the duplicate row detector used to implement DISTINCT. FossilOrigin-Name: d577dda0a7fbfacda57e8cad2bc4651d2a05d813 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 67b3b622ee..4e1872ccaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\s"PRAGMA\scase_sensitive_like"\sand\s"PRAGMA\sshrink_memory"\sset\sthe\nnumber\sof\soutput\scolumns\sto\s0\s(as\sthey\sare\sstatements\sthat\sreturn\sno\sdata). -D 2017-01-06T13:49:40.112 +C Avoid\sduplicate\sb-tree\ssearches\sin\sthe\sduplicate\srow\sdetector\sused\sto\nimplement\sDISTINCT. +D 2017-01-07T00:56:01.383 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -388,7 +388,7 @@ F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 4437d9d5d56b6ffdedabf394c7fe3a07ff521ce9 +F src/select.c 533e55a4067278fef76eff951462383d4147880f F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 F src/sqlite.h.in 29bda4bee01248a5650567d7a22fac39bad1b542 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 609ac1c73f7c2b48a571c178a72383996a9538fe -R 812971df62a9791f0bfb0122470a2213 -U dan -Z ebefd6b7654323e8a23a6cf648753c0a +P 6696cd1878be4bd44a24841b04163e52d847711e +R fe79848e4bc76903961662bd26c28411 +U drh +Z 1f3f3bb84e2a0b8cdaa2372e378cb7b9 diff --git a/manifest.uuid b/manifest.uuid index 865cc47649..034f53423a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6696cd1878be4bd44a24841b04163e52d847711e \ No newline at end of file +d577dda0a7fbfacda57e8cad2bc4651d2a05d813 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 75ae082e96..e750fd324d 100644 --- a/src/select.c +++ b/src/select.c @@ -657,6 +657,7 @@ static void codeDistinct( sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, r1); } From ad1d9a870742dc5525ca1e884bbdb8cc66dd45e6 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 7 Jan 2017 03:26:50 +0000 Subject: [PATCH 002/292] Avoid an unnecessary btree seek while deleting an index entry due to a conflict on a REPLACE operation. FossilOrigin-Name: f0495c5133d0dc04d63521136d6b9ca440792cdf --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/delete.c | 2 +- src/insert.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4e1872ccaf..c1d7de4c1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sduplicate\sb-tree\ssearches\sin\sthe\sduplicate\srow\sdetector\sused\sto\nimplement\sDISTINCT. -D 2017-01-07T00:56:01.383 +C Avoid\san\sunnecessary\sbtree\sseek\swhile\sdeleting\san\sindex\sentry\sdue\sto\sa\sconflict\non\sa\sREPLACE\soperation. +D 2017-01-07T03:26:50.193 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -340,7 +340,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c c8bc10d145c9666a34ae906250326fdaa8d58fa5 +F src/delete.c 1a443cedfff0420959416a09f55119973f803593 F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae @@ -350,7 +350,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 7af46a3be2656f5e13791464625d93d6b07b8612 +F src/insert.c ad2a0b2757a23b6f9297ee414eeab22b52fbde75 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6696cd1878be4bd44a24841b04163e52d847711e -R fe79848e4bc76903961662bd26c28411 +P d577dda0a7fbfacda57e8cad2bc4651d2a05d813 +R 9a736b4439c4e25faa8492f344a490a5 U drh -Z 1f3f3bb84e2a0b8cdaa2372e378cb7b9 +Z c755348d5a4bd2ab8fd825f60cd77850 diff --git a/manifest.uuid b/manifest.uuid index 034f53423a..c1cc73a62a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d577dda0a7fbfacda57e8cad2bc4651d2a05d813 \ No newline at end of file +f0495c5133d0dc04d63521136d6b9ca440792cdf \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 9ec8ed59e3..5a75e606e2 100644 --- a/src/delete.c +++ b/src/delete.c @@ -716,7 +716,7 @@ void sqlite3GenerateRowDelete( if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } - if( iIdxNoSeek>=0 ){ + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); } if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; diff --git a/src/insert.c b/src/insert.c index e261c1994e..e55ea2d9fb 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1651,7 +1651,7 @@ void sqlite3GenerateConstraintChecks( } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); seenReplace = 1; break; } From 4cef5b1c812cf5739251039d7d5d0e1223fe0caf Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 7 Jan 2017 14:26:28 +0000 Subject: [PATCH 003/292] Critical fix to the previous check-in so that it works when there are BEFORE triggers that move the cursor before the OP_Delete has a chance to be applied. FossilOrigin-Name: db2c0960ffb3b396b20e0441d3edb812254c82bc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/delete.c | 21 +++++++++------------ src/insert.c | 3 ++- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index c1d7de4c1b..769b92f991 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sunnecessary\sbtree\sseek\swhile\sdeleting\san\sindex\sentry\sdue\sto\sa\sconflict\non\sa\sREPLACE\soperation. -D 2017-01-07T03:26:50.193 +C Critical\sfix\sto\sthe\sprevious\scheck-in\sso\sthat\sit\sworks\swhen\sthere\sare\nBEFORE\striggers\sthat\smove\sthe\scursor\sbefore\sthe\sOP_Delete\shas\sa\schance\sto\nbe\sapplied. +D 2017-01-07T14:26:28.809 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -340,7 +340,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c 1a443cedfff0420959416a09f55119973f803593 +F src/delete.c 4220f4feee7fddefc1e73e5ade1ed82ffe8dc92a F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae @@ -350,7 +350,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c ad2a0b2757a23b6f9297ee414eeab22b52fbde75 +F src/insert.c 3b7fbb149ef249269bfbedabbe8fd706ef9a2317 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d577dda0a7fbfacda57e8cad2bc4651d2a05d813 -R 9a736b4439c4e25faa8492f344a490a5 +P f0495c5133d0dc04d63521136d6b9ca440792cdf +R 21f1a5c5760681e8d66bbbeed4342953 U drh -Z c755348d5a4bd2ab8fd825f60cd77850 +Z 06625003bcd408124313c9f36986c7f6 diff --git a/manifest.uuid b/manifest.uuid index c1cc73a62a..ce6b8d600a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0495c5133d0dc04d63521136d6b9ca440792cdf \ No newline at end of file +db2c0960ffb3b396b20e0441d3edb812254c82bc \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 5a75e606e2..db00803996 100644 --- a/src/delete.c +++ b/src/delete.c @@ -519,10 +519,7 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = -1; - if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ - iIdxNoSeek = aiCurOnePass[1]; - } + int iIdxNoSeek = bComplex ? -1 : aiCurOnePass[1]; sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } @@ -604,15 +601,15 @@ delete_from_cleanup: ** ** If eMode is ONEPASS_MULTI, then this call is being made as part ** of a ONEPASS delete that affects multiple rows. In this case, if -** iIdxNoSeek is a valid cursor number (>=0), then its position should -** be preserved following the delete operation. Or, if iIdxNoSeek is not -** a valid cursor number, the position of iDataCur should be preserved -** instead. +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. ** ** iIdxNoSeek: -** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an -** index cursor (from within array of cursors starting at iIdxCur) that -** already points to the index entry to be deleted. +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -683,7 +680,7 @@ void sqlite3GenerateRowDelete( /* If any BEFORE triggers were coded, then seek the cursor to the ** row to be deleted again. It may be that the BEFORE triggers moved - ** the cursor or of already deleted the row that the cursor was + ** the cursor or already deleted the row that the cursor was ** pointing to. */ if( addrStart Date: Sat, 7 Jan 2017 14:47:03 +0000 Subject: [PATCH 004/292] Improvements to the iIdxNoSeek optimization of sqlite3GenerateRowDelete() so that it is automatically disabled for BEFORE triggers but works in all other cases. FossilOrigin-Name: 3178ec4c27efc4ff84bcd17ddb17ec50a6ac96b3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/delete.c | 10 ++++++++-- src/insert.c | 3 +-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 769b92f991..ec76e312ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Critical\sfix\sto\sthe\sprevious\scheck-in\sso\sthat\sit\sworks\swhen\sthere\sare\nBEFORE\striggers\sthat\smove\sthe\scursor\sbefore\sthe\sOP_Delete\shas\sa\schance\sto\nbe\sapplied. -D 2017-01-07T14:26:28.809 +C Improvements\sto\sthe\siIdxNoSeek\soptimization\sof\ssqlite3GenerateRowDelete()\nso\sthat\sit\sis\sautomatically\sdisabled\sfor\sBEFORE\striggers\sbut\sworks\sin\sall\nother\scases. +D 2017-01-07T14:47:03.531 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -340,7 +340,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c 4220f4feee7fddefc1e73e5ade1ed82ffe8dc92a +F src/delete.c a84f6229ccb9448460c287248024ceb70e10baab F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae @@ -350,7 +350,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 3b7fbb149ef249269bfbedabbe8fd706ef9a2317 +F src/insert.c ad2a0b2757a23b6f9297ee414eeab22b52fbde75 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f0495c5133d0dc04d63521136d6b9ca440792cdf -R 21f1a5c5760681e8d66bbbeed4342953 +P db2c0960ffb3b396b20e0441d3edb812254c82bc +R 7bbc97d23d01181c7fdd8aa9c81894e1 U drh -Z 06625003bcd408124313c9f36986c7f6 +Z 5c5df114c2b0c58ce035451bb180f71f diff --git a/manifest.uuid b/manifest.uuid index ce6b8d600a..c040ce2769 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db2c0960ffb3b396b20e0441d3edb812254c82bc \ No newline at end of file +3178ec4c27efc4ff84bcd17ddb17ec50a6ac96b3 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index db00803996..1a5edb59d8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -519,9 +519,8 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = bComplex ? -1 : aiCurOnePass[1]; sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); } /* End of the loop over all rowids/primary-keys. */ @@ -610,6 +609,8 @@ delete_from_cleanup: ** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, ** then it identifies an index cursor (from within array of cursors ** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -682,11 +683,16 @@ void sqlite3GenerateRowDelete( ** row to be deleted again. It may be that the BEFORE triggers moved ** the cursor or already deleted the row that the cursor was ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. */ if( addrStart=0 ); + iIdxNoSeek = -1; } /* Do FK processing. This call checks that any FK constraints that diff --git a/src/insert.c b/src/insert.c index 4f407a2cbd..e55ea2d9fb 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1651,8 +1651,7 @@ void sqlite3GenerateConstraintChecks( } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), - (pTrigger ? -1 : iThisCur)); + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); seenReplace = 1; break; } From f4e994b23adc5bb62356af81d9fd194796bc22cf Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 9 Jan 2017 13:43:09 +0000 Subject: [PATCH 005/292] Add the SQLITE_UINT64_TYPE compile-time option. FossilOrigin-Name: a5fe03bc419d9c7e6068ed38810e3f183de179b5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ec76e312ce..9ab624a73c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\siIdxNoSeek\soptimization\sof\ssqlite3GenerateRowDelete()\nso\sthat\sit\sis\sautomatically\sdisabled\sfor\sBEFORE\striggers\sbut\sworks\sin\sall\nother\scases. -D 2017-01-07T14:47:03.531 +C Add\sthe\sSQLITE_UINT64_TYPE\scompile-time\soption. +D 2017-01-09T13:43:09.245 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -390,7 +390,7 @@ F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 533e55a4067278fef76eff951462383d4147880f F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 -F src/sqlite.h.in 29bda4bee01248a5650567d7a22fac39bad1b542 +F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h 9fdfb8789b27a621f3401468bc1705c32308f877 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db2c0960ffb3b396b20e0441d3edb812254c82bc -R 7bbc97d23d01181c7fdd8aa9c81894e1 +P 3178ec4c27efc4ff84bcd17ddb17ec50a6ac96b3 +R ff5d1e49d008400e89044e5b599d5e72 U drh -Z 5c5df114c2b0c58ce035451bb180f71f +Z 9ff193ccbc51dd8562e9e65994630a91 diff --git a/manifest.uuid b/manifest.uuid index c040ce2769..c450905317 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3178ec4c27efc4ff84bcd17ddb17ec50a6ac96b3 \ No newline at end of file +a5fe03bc419d9c7e6068ed38810e3f183de179b5 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1bcb62256c..7d7e70f901 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -259,7 +259,11 @@ typedef struct sqlite3 sqlite3; */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; - typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif #elif defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 sqlite_int64; typedef unsigned __int64 sqlite_uint64; From e7b554d615072f16896a6879f582c935fa252a12 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 9 Jan 2017 15:44:25 +0000 Subject: [PATCH 006/292] Modify the OP_RowData opcode so that when P3!=0 it is allowed to hold an ephemeral copy of the content. This avoids unnecessary memcpy() operations in the xfer-optimization and VACUUM. FossilOrigin-Name: 6e106acd74da3baa5c308a76443d2f0a7c904e5e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/insert.c | 4 ++-- src/vdbe.c | 27 +++++++++++++++++---------- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 9ab624a73c..5fc4d50822 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_UINT64_TYPE\scompile-time\soption. -D 2017-01-09T13:43:09.245 +C Modify\sthe\sOP_RowData\sopcode\sso\sthat\swhen\sP3!=0\sit\sis\sallowed\sto\shold\san\nephemeral\scopy\sof\sthe\scontent.\s\sThis\savoids\sunnecessary\smemcpy()\soperations\nin\sthe\sxfer-optimization\sand\sVACUUM. +D 2017-01-09T15:44:25.721 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -350,7 +350,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c ad2a0b2757a23b6f9297ee414eeab22b52fbde75 +F src/insert.c 7761fd63136771d411f096f4d7a1af9c5057ddd4 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -455,7 +455,7 @@ F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 88bd6c32b333580d2661ac3afe33369757fb1522 +F src/vdbe.c b7f39cec8b572df61cdbd26426e285f8fd759db3 F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3178ec4c27efc4ff84bcd17ddb17ec50a6ac96b3 -R ff5d1e49d008400e89044e5b599d5e72 +P a5fe03bc419d9c7e6068ed38810e3f183de179b5 +R 1f3e0de052f41632eb2fb92e1db69d89 U drh -Z 9ff193ccbc51dd8562e9e65994630a91 +Z 8ba5841c95779521fe8c2f52b8a5492f diff --git a/manifest.uuid b/manifest.uuid index c450905317..78d290dd0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5fe03bc419d9c7e6068ed38810e3f183de179b5 \ No newline at end of file +6e106acd74da3baa5c308a76443d2f0a7c904e5e \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index e55ea2d9fb..a33ee12353 100644 --- a/src/insert.c +++ b/src/insert.c @@ -2138,7 +2138,7 @@ static int xferOptimization( addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->flags & SQLITE_Vacuum ){ sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID| @@ -2170,7 +2170,7 @@ static int xferOptimization( sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->flags & SQLITE_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use diff --git a/src/vdbe.c b/src/vdbe.c index 5e707a6232..698092eb51 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4632,7 +4632,7 @@ case OP_SorterData: { break; } -/* Opcode: RowData P1 P2 * * * +/* Opcode: RowData P1 P2 P3 * * ** Synopsis: r[P2]=data ** ** Write into register P2 the complete row content for the row at @@ -4646,14 +4646,26 @@ case OP_SorterData: { ** ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. +** +** If P3!=0 then this opcode is allowed to make an ephermeral pointer +** into the database page. That means that the content of the output +** register will be invalidated as soon as the cursor moves - including +** moves caused by other cursors that "save" the the current cursors +** position in order that they can write to the same table. If P3==0 +** then a copy of the data is made into memory. P3!=0 is faster, but +** P3==0 is safer. +** +** If P3!=0 then the content of the P2 register is unsuitable for use +** in OP_Result and any OP_Result will invalidate the P2 register content. +** The P2 register content is invalided by opcodes like OP_Function or +** by any use of another cursor pointing to the same table. */ case OP_RowData: { VdbeCursor *pC; BtCursor *pCrsr; u32 n; - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -4684,14 +4696,9 @@ case OP_RowData: { goto too_big; } testcase( n==0 ); - if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ - goto no_mem; - } - pOut->n = n; - MemSetTypeFlag(pOut, MEM_Blob); - rc = sqlite3BtreePayload(pCrsr, 0, n, pOut->z); + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, n, pOut); if( rc ) goto abort_due_to_error; - pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ + if( !pOp->p3 ) Deephemeralize(pOut); UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); break; From ab61cf7d936bac8031d21803b6664cb4e5fc086e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 9 Jan 2017 18:22:54 +0000 Subject: [PATCH 007/292] Fix typo in a comment. No changes to code. FossilOrigin-Name: d38fd22935b1572f4481b39c2f9274329b18ea99 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5fc4d50822..01cee64cec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sOP_RowData\sopcode\sso\sthat\swhen\sP3!=0\sit\sis\sallowed\sto\shold\san\nephemeral\scopy\sof\sthe\scontent.\s\sThis\savoids\sunnecessary\smemcpy()\soperations\nin\sthe\sxfer-optimization\sand\sVACUUM. -D 2017-01-09T15:44:25.721 +C Fix\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2017-01-09T18:22:54.898 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -455,7 +455,7 @@ F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c b7f39cec8b572df61cdbd26426e285f8fd759db3 +F src/vdbe.c 5c7338cc8b28c76a9fb680e18640fee948ae29b2 F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a5fe03bc419d9c7e6068ed38810e3f183de179b5 -R 1f3e0de052f41632eb2fb92e1db69d89 -U drh -Z 8ba5841c95779521fe8c2f52b8a5492f +P 6e106acd74da3baa5c308a76443d2f0a7c904e5e +R bda8577547e21933c26bcd960cf7cb5b +U mistachkin +Z 7c89ce1d26860a2c4a6e46981671fc16 diff --git a/manifest.uuid b/manifest.uuid index 78d290dd0b..27218e961a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6e106acd74da3baa5c308a76443d2f0a7c904e5e \ No newline at end of file +d38fd22935b1572f4481b39c2f9274329b18ea99 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 698092eb51..3f165d9e89 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4657,7 +4657,7 @@ case OP_SorterData: { ** ** If P3!=0 then the content of the P2 register is unsuitable for use ** in OP_Result and any OP_Result will invalidate the P2 register content. -** The P2 register content is invalided by opcodes like OP_Function or +** The P2 register content is invalidated by opcodes like OP_Function or ** by any use of another cursor pointing to the same table. */ case OP_RowData: { From 7441df72be8d36089c31f21a6157cbbbdfe6d212 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 9 Jan 2017 19:27:04 +0000 Subject: [PATCH 008/292] Performance optimization and size reduction in the OP_Variable opcode. FossilOrigin-Name: 237aa97452e20c312f256a8fd62531e3d447f84b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 01cee64cec..10b70530cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2017-01-09T18:22:54.898 +C Performance\soptimization\sand\ssize\sreduction\sin\sthe\sOP_Variable\sopcode. +D 2017-01-09T19:27:04.729 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -455,7 +455,7 @@ F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 5c7338cc8b28c76a9fb680e18640fee948ae29b2 +F src/vdbe.c 51f0fb39cdd5f2307009887ff9aef075d5926a04 F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6e106acd74da3baa5c308a76443d2f0a7c904e5e -R bda8577547e21933c26bcd960cf7cb5b -U mistachkin -Z 7c89ce1d26860a2c4a6e46981671fc16 +P d38fd22935b1572f4481b39c2f9274329b18ea99 +R 7ff563d3f468514013639946cf62a576 +U drh +Z 47a0f13bc9d8903e7619705d6d0461a8 diff --git a/manifest.uuid b/manifest.uuid index 27218e961a..7009855f8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d38fd22935b1572f4481b39c2f9274329b18ea99 \ No newline at end of file +237aa97452e20c312f256a8fd62531e3d447f84b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 3f165d9e89..3f8dbd34c0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1195,7 +1195,7 @@ case OP_Variable: { /* out2 */ if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } - pOut = out2Prerelease(p, pOp); + pOut = &aMem[pOp->p2]; sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); UPDATE_MAX_BLOBSIZE(pOut); break; From e4a8b8769e38fa62bfbada282b4a07cc8aea4ece Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 9 Jan 2017 19:55:19 +0000 Subject: [PATCH 009/292] Remove a redundant assignment statement. FossilOrigin-Name: a5fa09657bd6c4ea5fe6712b0f8af2170cbe0381 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 10b70530cc..6f2234dddb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sand\ssize\sreduction\sin\sthe\sOP_Variable\sopcode. -D 2017-01-09T19:27:04.729 +C Remove\sa\sredundant\sassignment\sstatement. +D 2017-01-09T19:55:19.701 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -455,7 +455,7 @@ F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 51f0fb39cdd5f2307009887ff9aef075d5926a04 +F src/vdbe.c 4c239b73d8df6ccd82842e2de0a882be46f6152d F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d38fd22935b1572f4481b39c2f9274329b18ea99 -R 7ff563d3f468514013639946cf62a576 +P 237aa97452e20c312f256a8fd62531e3d447f84b +R 3d5f5ba0e62157717b12def8c248d9db U drh -Z 47a0f13bc9d8903e7619705d6d0461a8 +Z 6d8150e7f23763db279bf09c6fea02eb diff --git a/manifest.uuid b/manifest.uuid index 7009855f8c..5e3236a559 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -237aa97452e20c312f256a8fd62531e3d447f84b \ No newline at end of file +a5fa09657bd6c4ea5fe6712b0f8af2170cbe0381 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 3f8dbd34c0..7f6f21b6ea 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5208,7 +5208,6 @@ case OP_IdxRowid: { /* out2 */ }else{ pOut = out2Prerelease(p, pOp); pOut->u.i = rowid; - pOut->flags = MEM_Int; } }else{ assert( pOp->opcode==OP_IdxRowid ); From d9bcb32ebb4b94df6c241bc09080cd0832d3a01b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 10 Jan 2017 15:08:06 +0000 Subject: [PATCH 010/292] Fix a potential assertion fault discovered by OSS-Fuzz. FossilOrigin-Name: 71c03b59b645884ebd6b9e18713cd2eb8c949870 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6f2234dddb..263f2b60cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sredundant\sassignment\sstatement. -D 2017-01-09T19:55:19.701 +C Fix\sa\spotential\sassertion\sfault\sdiscovered\sby\sOSS-Fuzz. +D 2017-01-10T15:08:06.289 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -472,7 +472,7 @@ F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 6bbf9284f4f15a6fa48663d033870cc0d7f5ee66 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 -F src/whereexpr.c 87ecdf24beba4498e4380b31c4131febb0a6ceaa +F src/whereexpr.c 24e452bcc36ac19130706357bbec4c1419931222 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 237aa97452e20c312f256a8fd62531e3d447f84b -R 3d5f5ba0e62157717b12def8c248d9db +P a5fa09657bd6c4ea5fe6712b0f8af2170cbe0381 +R 9d11b71201b21d28f9619103dd00f618 U drh -Z 6d8150e7f23763db279bf09c6fea02eb +Z 82e96cf4e2e496f2e42165b862cbc10e diff --git a/manifest.uuid b/manifest.uuid index 5e3236a559..9a414db70b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5fa09657bd6c4ea5fe6712b0f8af2170cbe0381 \ No newline at end of file +71c03b59b645884ebd6b9e18713cd2eb8c949870 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 4bb161044e..27c6ebc2f1 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -913,6 +913,7 @@ static void exprAnalyze( Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + int nLeft; /* Number of elements on left side vector */ if( db->mallocFailed ){ return; @@ -1184,13 +1185,12 @@ static void exprAnalyze( ** is not a sub-select. */ if( pWC->op==TK_AND && (pExpr->op==TK_EQ || pExpr->op==TK_IS) - && sqlite3ExprIsVector(pExpr->pLeft) + && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 + && sqlite3ExprVectorSize(pExpr->pRight)==nLeft && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 - || (pExpr->pRight->flags & EP_xIsSelect)==0 - )){ - int nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + || (pExpr->pRight->flags & EP_xIsSelect)==0) + ){ int i; - assert( nLeft==sqlite3ExprVectorSize(pExpr->pRight) ); for(i=0; i Date: Tue, 10 Jan 2017 16:09:46 +0000 Subject: [PATCH 011/292] Avoid unnecessary calls to the xRoundup() method of the memory allocator when the soft heap limit is not set. FossilOrigin-Name: 4209b89eab01814228a178963238e0dffffad2a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 12 +++++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 263f2b60cf..d85a6d5b2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sassertion\sfault\sdiscovered\sby\sOSS-Fuzz. -D 2017-01-10T15:08:06.289 +C Avoid\sunnecessary\scalls\sto\sthe\sxRoundup()\smethod\sof\sthe\smemory\sallocator\swhen\nthe\ssoft\sheap\slimit\sis\snot\sset. +D 2017-01-10T16:09:46.718 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -354,7 +354,7 @@ F src/insert.c 7761fd63136771d411f096f4d7a1af9c5057ddd4 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c f3fad34cd570022abca558c573f1761fb09a8212 +F src/malloc.c c36ef8fa6e4cc53ec258c5aa3ad86eb47350cc3d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a5fa09657bd6c4ea5fe6712b0f8af2170cbe0381 -R 9d11b71201b21d28f9619103dd00f618 +P 71c03b59b645884ebd6b9e18713cd2eb8c949870 +R 631021f153b5b713372ffbd6126c74cb U drh -Z 82e96cf4e2e496f2e42165b862cbc10e +Z 02d94bfa17296c03c0e8153386f8771f diff --git a/manifest.uuid b/manifest.uuid index 9a414db70b..5b72849afa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71c03b59b645884ebd6b9e18713cd2eb8c949870 \ No newline at end of file +4209b89eab01814228a178963238e0dffffad2a4 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 84191c78a1..053b57b47f 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -217,14 +217,13 @@ static void sqlite3MallocAlarm(int nByte){ ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. */ -static int mallocWithAlarm(int n, void **pp){ - int nFull; +static void mallocWithAlarm(int n, void **pp){ void *p; assert( sqlite3_mutex_held(mem0.mutex) ); - nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + int nFull = sqlite3GlobalConfig.m.xRoundup(n); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -232,20 +231,19 @@ static int mallocWithAlarm(int n, void **pp){ mem0.nearlyFull = 0; } } - p = sqlite3GlobalConfig.m.xMalloc(nFull); + p = sqlite3GlobalConfig.m.xMalloc(n); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); - p = sqlite3GlobalConfig.m.xMalloc(nFull); + p = sqlite3GlobalConfig.m.xMalloc(n); } #endif if( p ){ - nFull = sqlite3MallocSize(p); + int nFull = sqlite3MallocSize(p); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; - return nFull; } /* From 8e36ddd37e5d664c30c20fb9f2e238229f4ad078 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 10 Jan 2017 17:33:43 +0000 Subject: [PATCH 012/292] Throw an error if the ON clause of a LEFT JOIN references tables to the right of the ON clause. Fix for ticket [25e335f802dd]. FossilOrigin-Name: c92ecff2ec5f178433d21f25c653d0fdd9128d7c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d85a6d5b2c..e94ce3d85b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\scalls\sto\sthe\sxRoundup()\smethod\sof\sthe\smemory\sallocator\swhen\nthe\ssoft\sheap\slimit\sis\snot\sset. -D 2017-01-10T16:09:46.718 +C Throw\san\serror\sif\sthe\sON\sclause\sof\sa\sLEFT\sJOIN\sreferences\stables\sto\sthe\sright\nof\sthe\sON\sclause.\s\sFix\sfor\sticket\s[25e335f802dd]. +D 2017-01-10T17:33:43.685 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -472,7 +472,7 @@ F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 6bbf9284f4f15a6fa48663d033870cc0d7f5ee66 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 -F src/whereexpr.c 24e452bcc36ac19130706357bbec4c1419931222 +F src/whereexpr.c 35ad025389a632a3987a35617c878be3b3d70dc6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71c03b59b645884ebd6b9e18713cd2eb8c949870 -R 631021f153b5b713372ffbd6126c74cb +P 4209b89eab01814228a178963238e0dffffad2a4 +R f2db398370e2f9344a20853ca8cb314d U drh -Z 02d94bfa17296c03c0e8153386f8771f +Z 014b9abce2c0eec156f7bce8e0868d39 diff --git a/manifest.uuid b/manifest.uuid index 5b72849afa..907b8ba384 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4209b89eab01814228a178963238e0dffffad2a4 \ No newline at end of file +c92ecff2ec5f178433d21f25c653d0fdd9128d7c \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 27c6ebc2f1..826d329b7f 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -943,6 +943,10 @@ static void exprAnalyze( prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ + if( (prereqAll>>1)>=x ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } } pTerm->prereqAll = prereqAll; pTerm->leftCursor = -1; From f112f0b3deb251173c6695769bf2f622bbaadfb6 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Jan 2017 17:37:49 +0000 Subject: [PATCH 013/292] Add a test case for ticket [25e335f802dd]. FossilOrigin-Name: e500c15a9f55aed1601f7c14169dd56fd76f1fdd --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/join2.test | 22 +++++++++++++++++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e94ce3d85b..d7b4aa7e62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Throw\san\serror\sif\sthe\sON\sclause\sof\sa\sLEFT\sJOIN\sreferences\stables\sto\sthe\sright\nof\sthe\sON\sclause.\s\sFix\sfor\sticket\s[25e335f802dd]. -D 2017-01-10T17:33:43.685 +C Add\sa\stest\scase\sfor\sticket\s[25e335f802dd]. +D 2017-01-10T17:37:49.188 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -880,7 +880,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test f9d4a28dec81c6e9dc21b73518e024d73b5ebf57 -F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 +F test/join2.test a48f723c5692e2cbb23a9297ac2720cb77d51a70 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4209b89eab01814228a178963238e0dffffad2a4 -R f2db398370e2f9344a20853ca8cb314d -U drh -Z 014b9abce2c0eec156f7bce8e0868d39 +P c92ecff2ec5f178433d21f25c653d0fdd9128d7c +R e5ba5e796b740bb6ab62d42fde094373 +U dan +Z ef9b8c72a2a40eee5e78f0752f5e6ce0 diff --git a/manifest.uuid b/manifest.uuid index 907b8ba384..34fbf06d32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c92ecff2ec5f178433d21f25c653d0fdd9128d7c \ No newline at end of file +e500c15a9f55aed1601f7c14169dd56fd76f1fdd \ No newline at end of file diff --git a/test/join2.test b/test/join2.test index 0f558c5a3d..9372e770c3 100644 --- a/test/join2.test +++ b/test/join2.test @@ -12,10 +12,10 @@ # # This file implements tests for joins, including outer joins. # -# $Id: join2.test,v 1.2 2005/01/21 03:12:16 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix join2 do_test join2-1.1 { execsql { @@ -72,4 +72,24 @@ ifcapable subquery { } {1 11 111 1111 2 22 {} {} 3 33 {} {}} } +#------------------------------------------------------------------------- +# Check that ticket [25e335f802ddc] has been resolved. It should be an +# error for the ON clause of a LEFT JOIN to refer to a table to its right. +# +do_execsql_test 2.0 { + CREATE TABLE aa(a); + CREATE TABLE bb(b); + CREATE TABLE cc(c); + INSERT INTO aa VALUES('one'); + INSERT INTO bb VALUES('one'); + INSERT INTO cc VALUES('one'); +} + +do_catchsql_test 2.1 { + SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=c); +} {1 {ON clause references tables to its right}} +do_catchsql_test 2.2 { + SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c); +} {0 {one one one}} + finish_test From f91c1318f48febae7d3aac6c14b70068f1a2c375 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Jan 2017 20:04:38 +0000 Subject: [PATCH 014/292] Changes to allow some multi-row UPDATE statements to avoid the two-pass approach. FossilOrigin-Name: 46db23ccd116ce5b9d949f9293be8a2818411b46 --- manifest | 30 ++++++----- manifest.uuid | 2 +- src/btree.c | 31 +++++++++-- src/btree.h | 5 +- src/insert.c | 10 +++- src/select.c | 2 +- src/sqliteInt.h | 4 +- src/update.c | 129 +++++++++++++++++++++++++++++----------------- src/vdbe.c | 4 +- src/where.c | 3 +- test/update2.test | 84 ++++++++++++++++++++++++++++++ 11 files changed, 228 insertions(+), 76 deletions(-) create mode 100644 test/update2.test diff --git a/manifest b/manifest index d7b4aa7e62..e5b56745a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sticket\s[25e335f802dd]. -D 2017-01-10T17:37:49.188 +C Changes\sto\sallow\ssome\smulti-row\sUPDATE\sstatements\sto\savoid\sthe\stwo-pass\napproach. +D 2017-01-10T20:04:38.186 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -331,8 +331,8 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c d2c100618784bd89c089fcef03ff6e789768ecae -F src/btree.h 2349a588abcd7e0c04f984e15c5c777b61637583 +F src/btree.c 44e9612965f63bef288673b81faa43e765bcac5f +F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 @@ -350,7 +350,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 7761fd63136771d411f096f4d7a1af9c5057ddd4 +F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -388,12 +388,12 @@ F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 533e55a4067278fef76eff951462383d4147880f +F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 9fdfb8789b27a621f3401468bc1705c32308f877 +F src/sqliteInt.h bec6274d8991528bc12d9a34d01fe84bdf6d00d9 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -451,11 +451,11 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 1da7c462110bffed442a42884cb0d528c1db46d8 +F src/update.c 77a02122e040b8c6e1f13db9ef203e2224dae8f8 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 4c239b73d8df6ccd82842e2de0a882be46f6152d +F src/vdbe.c c7add5978cb84ae3a7bcb16f8b56cb3bbdf04b7e F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 @@ -469,7 +469,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 6bbf9284f4f15a6fa48663d033870cc0d7f5ee66 +F src/where.c bc71775e23d23334e8f449aa31012d692dc09cb2 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 F src/whereexpr.c 35ad025389a632a3987a35617c878be3b3d70dc6 @@ -1351,6 +1351,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 +F test/update2.test e7f23b8ed101b7113c198bbd6f78f508718725c1 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 @@ -1543,7 +1544,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c92ecff2ec5f178433d21f25c653d0fdd9128d7c -R e5ba5e796b740bb6ab62d42fde094373 +P e500c15a9f55aed1601f7c14169dd56fd76f1fdd +R 210f82ca33b63d0d25de1e8cfd9403fd +T *branch * onepass-update +T *sym-onepass-update * +T -sym-trunk * U dan -Z ef9b8c72a2a40eee5e78f0752f5e6ce0 +Z 00c5ac8bc1c536bfb67f723788417749 diff --git a/manifest.uuid b/manifest.uuid index 34fbf06d32..00d064c0e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e500c15a9f55aed1601f7c14169dd56fd76f1fdd \ No newline at end of file +46db23ccd116ce5b9d949f9293be8a2818411b46 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f869761625..60f886bf5a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7948,7 +7948,7 @@ static int balance(BtCursor *pCur){ int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ const BtreePayload *pX, /* Content of the row to be inserted */ - int appendBias, /* True if this is likely an append */ + int flags, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ int rc; @@ -7961,6 +7961,8 @@ int sqlite3BtreeInsert( unsigned char *oldCell; unsigned char *newCell = 0; + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND))==flags ); + if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); return pCur->skipNext; @@ -8001,6 +8003,11 @@ int sqlite3BtreeInsert( ** cursors open on the row being replaced */ invalidateIncrblobCursors(p, pX->nKey, 0); + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. */ + assert( (flags & BTREE_SAVEPOSITION)==0 || + ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) ); + /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary ** btreeMoveto() call */ @@ -8010,10 +8017,10 @@ int sqlite3BtreeInsert( && pCur->info.nKey==pX->nKey-1 ){ loc = -1; }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); if( rc ) return rc; } - }else if( loc==0 ){ + }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ if( pX->nMem ){ UnpackedRecord r; r.pKeyInfo = pCur->pKeyInfo; @@ -8024,9 +8031,9 @@ int sqlite3BtreeInsert( r.r1 = 0; r.r2 = 0; r.eqSeen = 0; - rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, flags!=0, &loc); }else{ - rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc); + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, flags!=0, &loc); } if( rc ) return rc; } @@ -8114,6 +8121,20 @@ int sqlite3BtreeInsert( ** from trying to save the current position of the cursor. */ pCur->apPage[pCur->iPage]->nOverflow = 0; pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + rc = moveToRoot(pCur); + if( pCur->pKeyInfo && rc==SQLITE_OK ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } } assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); diff --git a/src/btree.h b/src/btree.h index 5e54125d39..ae57468e3f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -249,9 +249,10 @@ int sqlite3BtreeCursorHasMoved(BtCursor*); int sqlite3BtreeCursorRestore(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*, u8 flags); -/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ #define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ #define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ /* An instance of the BtreePayload object describes the content of a single ** entry in either an index or table btree. @@ -282,7 +283,7 @@ struct BtreePayload { }; int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, - int bias, int seekResult); + int flags, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int *pRes); diff --git a/src/insert.c b/src/insert.c index a33ee12353..93c22ae3f5 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1684,7 +1684,7 @@ void sqlite3CompleteInsertion( int iIdxCur, /* First index cursor */ int regNewData, /* Range of content */ int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int isUpdate, /* True for UPDATE, False for INSERT */ + int update_flags, /* True for UPDATE, False for INSERT */ int appendBias, /* True if this is likely to be an append */ int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ @@ -1696,6 +1696,11 @@ void sqlite3CompleteInsertion( int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1714,6 +1719,7 @@ void sqlite3CompleteInsertion( if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -1729,7 +1735,7 @@ void sqlite3CompleteInsertion( pik_flags = 0; }else{ pik_flags = OPFLAG_NCHANGE; - pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); } if( appendBias ){ pik_flags |= OPFLAG_APPEND; diff --git a/src/select.c b/src/select.c index e750fd324d..ed6221309f 100644 --- a/src/select.c +++ b/src/select.c @@ -5652,7 +5652,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8cfa0f88f7..3215a7f865 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3041,7 +3041,7 @@ struct AuthContext { #define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ /* Also used in P2 (not P5) of OP_Delete */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ -#define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ +#define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ @@ -3055,7 +3055,7 @@ struct AuthContext { #define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ #define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ -#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete/Insert: save cursor pos */ #define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ /* diff --git a/src/update.c b/src/update.c index 5f89e31dac..c99a4b6e8c 100644 --- a/src/update.c +++ b/src/update.c @@ -105,7 +105,7 @@ void sqlite3Update( int iDataCur; /* Cursor for the canonical data btree */ int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* One register assigned to each index to be updated */ + int *aRegIdx = 0; /* First register in array assigned to each index */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ @@ -117,10 +117,11 @@ void sqlite3Update( AuthContext sContext; /* The authorization context */ NameContext sNC; /* The name-context to resolve expressions in */ int iDb; /* Database containing the table being updated */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_XXX value from where.c */ int hasFK; /* True if foreign key processing is required */ int labelBreak; /* Jump here to break out of UPDATE loop */ int labelContinue; /* Jump here to continue next step of UPDATE loop */ + int flags; /* Flags for sqlite3WhereBegin() */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True when updating a view (INSTEAD OF trigger) */ @@ -131,6 +132,9 @@ void sqlite3Update( int iEph = 0; /* Ephemeral table holding all primary key values */ int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + int addrOpen; /* Address of OP_OpenEphemeral */ + int iPk; /* First of nPk cells holding PRIMARY KEY value */ + i16 nPk; /* Number of components of the PRIMARY KEY */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -349,51 +353,70 @@ void sqlite3Update( } #endif - /* Begin the database scan - */ + /* Initialize the count of updated rows */ + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); - pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur - ); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - - /* Remember the rowid of every item to be updated. - */ - sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); - if( !okOnePass ){ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); - } - - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); }else{ - int iPk; /* First of nPk memory cells holding PRIMARY KEY value */ - i16 nPk; /* Number of components of the PRIMARY KEY */ - int addrOpen; /* Address of the OpenEphemeral instruction */ - assert( pPk!=0 ); nPk = pPk->nKeyCol; iPk = pParse->nMem+1; pParse->nMem += nPk; regKey = ++pParse->nMem; iEph = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); sqlite3VdbeSetP4KeyInfo(pParse, pPk); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED, iIdxCur); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + } + + /* Begin the database scan. */ + flags = WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE; + if( pParse->nested==0 && pTrigger==0 && hasFK==0 && chngKey==0 ){ + flags |= WHERE_ONEPASS_MULTIROW; + } + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + + /* A one-pass strategy that might update more than one row may not + ** be used if any column of the index used for the scan is being + ** updated. Otherwise, if there is an index on "b", statements like + ** the following could create an infinite loop: + ** + ** UPDATE t1 SET b=b+1 WHERE b>? + ** + ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI + ** strategy that uses an index for which one or more columns are being + ** updated. */ + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + if( eOnePass==ONEPASS_MULTI ){ + int iCur = aiCurOnePass[1]; + if( iCur>=0 && aToOpen[iCur-iBaseCur] ) eOnePass = ONEPASS_OFF; + } + + if( HasRowid(pTab) ){ + /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF + ** mode, write the rowid into the FIFO. In either of the one-pass modes, + ** leave it in register regOldRowid. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( eOnePass==ONEPASS_OFF ){ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + } + }else{ + /* Read the PK of the current row into an array of registers. In + ** ONEPASS_OFF mode, serialize the array into a record and store it in + ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change + ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table + ** is not required) and leave the PK fields in the array of registers. */ for(i=0; iaiColumn[i]>=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], - iPk+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i); } - if( okOnePass ){ + if( eOnePass ){ sqlite3VdbeChangeToNoop(v, addrOpen); nKey = nPk; regKey = iPk; @@ -402,18 +425,15 @@ void sqlite3Update( sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk); } - sqlite3WhereEnd(pWInfo); } - /* Initialize the count of updated rows - */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ - regRowCount = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + if( eOnePass!=ONEPASS_MULTI ){ + sqlite3WhereEnd(pWInfo); } labelBreak = sqlite3VdbeMakeLabel(v); if( !isView ){ + int iAddrOnce = 0; /* ** Open every index that needs updating. Note that if any ** index could potentially invoke a REPLACE conflict resolution @@ -430,22 +450,31 @@ void sqlite3Update( } } } - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } + + if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){ + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); + if( iAddrOnce ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Top of the update loop */ - if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] && !isView ){ + if( eOnePass!=ONEPASS_OFF ){ + if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){ assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } - labelContinue = labelBreak; + if( eOnePass==ONEPASS_SINGLE ){ + labelContinue = labelBreak; + }else{ + labelContinue = sqlite3VdbeMakeLabel(v); + } sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); VdbeCoverageIf(v, pPk==0); VdbeCoverageIf(v, pPk!=0); @@ -606,14 +635,14 @@ void sqlite3Update( assert( regNew==regNewRowid+1 ); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, - OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP), + OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP), regNewRowid ); if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } #else - if( hasFK || chngKey || pPk!=0 ){ + if( hasFK || chngKey ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif @@ -626,8 +655,11 @@ void sqlite3Update( } /* Insert the new index entries and the new record. */ - sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, - regNewRowid, aRegIdx, 1, 0, 0); + sqlite3CompleteInsertion( + pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, + OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), + 0, 0 + ); /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key @@ -649,8 +681,11 @@ void sqlite3Update( /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. */ - if( okOnePass ){ + if( eOnePass==ONEPASS_SINGLE ){ /* Nothing to do at end-of-loop for a single-pass */ + }else if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); diff --git a/src/vdbe.c b/src/vdbe.c index 7f6f21b6ea..cbb7867512 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4421,7 +4421,7 @@ case OP_InsertInt: { } x.pKey = 0; rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, - (pOp->p5 & OPFLAG_APPEND)!=0, seekResult + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), seekResult ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -5086,7 +5086,7 @@ case OP_IdxInsert: { /* in2 */ x.aMem = aMem + pOp->p3; x.nMem = (u16)pOp->p4.i; rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, - (pOp->p5 & OPFLAG_APPEND)!=0, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) ); assert( pC->deferredMoveto==0 ); diff --git a/src/where.c b/src/where.c index 81cc1f131b..4951f6a1b4 100644 --- a/src/where.c +++ b/src/where.c @@ -4949,7 +4949,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 + || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; diff --git a/test/update2.test b/test/update2.test new file mode 100644 index 0000000000..fad31ec423 --- /dev/null +++ b/test/update2.test @@ -0,0 +1,84 @@ +# 2017 January 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix update2 + +db func repeat [list string repeat] + +#------------------------------------------------------------------------- +# 1.1.* A one-pass UPDATE that does balance() operations on the IPK index +# that it is scanning. +# +# 1.2.* Same again, but with a WITHOUT ROWID table. +# +set nrow [expr 10] +do_execsql_test 1.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow ) + INSERT INTO t1(b) SELECT char((i % 26) + 65) FROM s; + INSERT INTO t2 SELECT * FROM t1; +} + +do_execsql_test 1.1.1 { + UPDATE t1 SET b = repeat(b, 100) +} + +do_execsql_test 1.1.2 { + SELECT * FROM t1; +} [db eval { SELECT a, repeat(b, 100) FROM t2 }] + +do_execsql_test 1.2.0 { + DROP TABLE t1; + CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID; + WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow ) + INSERT INTO t1(a, b) SELECT i+1, char((i % 26) + 65) FROM s; +} + +#explain_i { UPDATE t1 SET b = repeat(b, 100) } +do_execsql_test 1.2.1 { + UPDATE t1 SET b = repeat(b, 100) +} + +do_execsql_test 1.2.2 { + SELECT * FROM t1; +} [db eval { SELECT a, repeat(b, 100) FROM t2 }] + + +#------------------------------------------------------------------------- +# A one-pass UPDATE that does balance() operations on the IPK index +# that it is scanning. +# +do_execsql_test 2.1 { + CREATE TABLE t3(a PRIMARY KEY, b, c); + CREATE INDEX t3i ON t3(b); +} {} +do_execsql_test 2.2 { UPDATE t3 SET c=1 WHERE b=? } {} +do_execsql_test 2.3 { UPDATE t3 SET c=1 WHERE rowid=? } {} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + CREATE TABLE t4(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX t4c ON t4(c); + INSERT INTO t4 VALUES(1, 2, 3); + INSERT INTO t4 VALUES(2, 3, 4); +} + +do_execsql_test 3.1 { + UPDATE t4 SET c=c+2 WHERE c>2; + SELECT a, c FROM t4 ORDER BY a; +} {1 5 2 6} + +finish_test From 785d8ed0d4d5f038b6731420b3daff1798711b48 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Jan 2017 14:15:29 +0000 Subject: [PATCH 015/292] In the STAT4 computations, ensure that the aAvgEq values do not go negative. FossilOrigin-Name: f58f75b5a06f88ba97bd1a02bee621c64691c6f8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d7b4aa7e62..60b608e6c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sticket\s[25e335f802dd]. -D 2017-01-10T17:37:49.188 +C In\sthe\sSTAT4\scomputations,\sensure\sthat\sthe\saAvgEq\svalues\sdo\snot\sgo\snegative. +D 2017-01-11T14:15:29.635 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -325,7 +325,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 3c4a63ff7a55faefecf6eb1589932fdbc06b2415 +F src/analyze.c 317dbaf31c16050582b09bf4f323b4e0f1813251 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -1543,7 +1543,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c92ecff2ec5f178433d21f25c653d0fdd9128d7c -R e5ba5e796b740bb6ab62d42fde094373 -U dan -Z ef9b8c72a2a40eee5e78f0752f5e6ce0 +P e500c15a9f55aed1601f7c14169dd56fd76f1fdd +R 5c2e2ea0b7d53513cb0d1e5b25773986 +U drh +Z ed5473ac37dfd76118f46ef5f94f7fb6 diff --git a/manifest.uuid b/manifest.uuid index 34fbf06d32..1657a590c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e500c15a9f55aed1601f7c14169dd56fd76f1fdd \ No newline at end of file +f58f75b5a06f88ba97bd1a02bee621c64691c6f8 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c480a0c507..c079132f4a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1615,7 +1615,7 @@ static void initAvgEq(Index *pIdx){ } } - if( nDist100>nSum100 ){ + if( nDist100>nSum100 && sumEq Date: Wed, 11 Jan 2017 15:42:14 +0000 Subject: [PATCH 016/292] Fix a problem preventing UPDATE statements that use a range-scan on the PK index of a WITHOUT ROWID table from using a one-pass strategy. FossilOrigin-Name: cab86c90945126c24c40cf2dedd053a8c46d00d6 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/update.c | 5 ++++- test/update2.test | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index e5b56745a1..5190754865 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sallow\ssome\smulti-row\sUPDATE\sstatements\sto\savoid\sthe\stwo-pass\napproach. -D 2017-01-10T20:04:38.186 +C Fix\sa\sproblem\spreventing\sUPDATE\sstatements\sthat\suse\sa\srange-scan\son\sthe\sPK\nindex\sof\sa\sWITHOUT\sROWID\stable\sfrom\susing\sa\sone-pass\sstrategy. +D 2017-01-11T15:42:14.353 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -451,7 +451,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 77a02122e040b8c6e1f13db9ef203e2224dae8f8 +F src/update.c 117bb080654b8c382b590b369f174ec11cca5bb0 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -1351,7 +1351,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 -F test/update2.test e7f23b8ed101b7113c198bbd6f78f508718725c1 +F test/update2.test 2b678973f882d11496f998c01b310be4c4a65a70 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 @@ -1544,10 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e500c15a9f55aed1601f7c14169dd56fd76f1fdd -R 210f82ca33b63d0d25de1e8cfd9403fd -T *branch * onepass-update -T *sym-onepass-update * -T -sym-trunk * +P 46db23ccd116ce5b9d949f9293be8a2818411b46 +R a3998d49adecb7e21ad8d0444ff44124 U dan -Z 00c5ac8bc1c536bfb67f723788417749 +Z c5a6bc3f18beb290c2a3c904ee96a0c7 diff --git a/manifest.uuid b/manifest.uuid index 00d064c0e5..294d40d293 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46db23ccd116ce5b9d949f9293be8a2818411b46 \ No newline at end of file +cab86c90945126c24c40cf2dedd053a8c46d00d6 \ No newline at end of file diff --git a/src/update.c b/src/update.c index c99a4b6e8c..acf43034ff 100644 --- a/src/update.c +++ b/src/update.c @@ -395,7 +395,10 @@ void sqlite3Update( eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); if( eOnePass==ONEPASS_MULTI ){ int iCur = aiCurOnePass[1]; - if( iCur>=0 && aToOpen[iCur-iBaseCur] ) eOnePass = ONEPASS_OFF; + if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){ + eOnePass = ONEPASS_OFF; + } + assert( iCur!=iDataCur || !HasRowid(pTab) ); } if( HasRowid(pTab) ){ diff --git a/test/update2.test b/test/update2.test index fad31ec423..8aafb2438b 100644 --- a/test/update2.test +++ b/test/update2.test @@ -81,4 +81,38 @@ do_execsql_test 3.1 { SELECT a, c FROM t4 ORDER BY a; } {1 5 2 6} +#------------------------------------------------------------------------- +# +foreach {tn sql} { + 1 { CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c) } + 2 { CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } +} { + execsql { DROP TABLE IF EXISTS b1 } + execsql $sql + do_execsql_test 4.$tn.0 { + CREATE UNIQUE INDEX b1c ON b1(c); + + INSERT INTO b1 VALUES(1, 'a', 1); + INSERT INTO b1 VALUES(2, 'b', 15); + INSERT INTO b1 VALUES(3, 'c', 3); + INSERT INTO b1 VALUES(4, 'd', 4); + INSERT INTO b1 VALUES(5, 'e', 5); + INSERT INTO b1 VALUES(6, 'f', 6); + INSERT INTO b1 VALUES(7, 'g', 7); + } + + do_execsql_test 4.$tn.1 { + UPDATE OR REPLACE b1 SET c=c+10 WHERE a BETWEEN 4 AND 7; + SELECT * FROM b1 ORDER BY a; + } { + 1 a 1 + 3 c 3 + 4 d 14 + 5 e 15 + 6 f 16 + 7 g 17 + } +} + + finish_test From 2c6fec21dc38810eacc702467f2b909f7325608f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Jan 2017 19:03:08 +0000 Subject: [PATCH 017/292] Fix a problem with single-pass multi-row UPDATE statements that invoke REPLACE conflict handling. FossilOrigin-Name: 0a2b8e1b9dc600b5a93622e8eea6218649df5e0f --- manifest | 14 ++++----- manifest.uuid | 2 +- src/update.c | 46 +++++++++++++++--------------- test/update2.test | 72 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 5190754865..95612d61fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sUPDATE\sstatements\sthat\suse\sa\srange-scan\son\sthe\sPK\nindex\sof\sa\sWITHOUT\sROWID\stable\sfrom\susing\sa\sone-pass\sstrategy. -D 2017-01-11T15:42:14.353 +C Fix\sa\sproblem\swith\ssingle-pass\smulti-row\sUPDATE\sstatements\sthat\sinvoke\sREPLACE\nconflict\shandling. +D 2017-01-11T19:03:08.308 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -451,7 +451,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 117bb080654b8c382b590b369f174ec11cca5bb0 +F src/update.c 4ed0fcccccd6488ee8a3bc02df10979b503f25d7 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -1351,7 +1351,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 -F test/update2.test 2b678973f882d11496f998c01b310be4c4a65a70 +F test/update2.test 7f72ef307d58be09a8ef61db23a7ef60e5af019d F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46db23ccd116ce5b9d949f9293be8a2818411b46 -R a3998d49adecb7e21ad8d0444ff44124 +P cab86c90945126c24c40cf2dedd053a8c46d00d6 +R 91b60fc45aced248f038aee86694df0e U dan -Z c5a6bc3f18beb290c2a3c904ee96a0c7 +Z f69bdc261645bb06f02a001c239441ef diff --git a/manifest.uuid b/manifest.uuid index 294d40d293..c670eb3052 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cab86c90945126c24c40cf2dedd053a8c46d00d6 \ No newline at end of file +0a2b8e1b9dc600b5a93622e8eea6218649df5e0f \ No newline at end of file diff --git a/src/update.c b/src/update.c index acf43034ff..242d36b739 100644 --- a/src/update.c +++ b/src/update.c @@ -135,6 +135,7 @@ void sqlite3Update( int addrOpen; /* Address of OP_OpenEphemeral */ int iPk; /* First of nPk cells holding PRIMARY KEY value */ i16 nPk; /* Number of components of the PRIMARY KEY */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -294,6 +295,11 @@ void sqlite3Update( if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; pParse->nMem += pIdx->nColumn; + if( (onError==OE_Replace) + || (onError==OE_Default && pIdx->onError==OE_Replace) + ){ + bReplace = 1; + } break; } } @@ -301,6 +307,11 @@ void sqlite3Update( if( reg==0 ) aToOpen[j+1] = 0; aRegIdx[j] = reg; } + if( bReplace ){ + /* If REPLACE conflict resolution might be invoked, open cursors on all + ** indexes in case they are needed to delete records. */ + memset(aToOpen, 1, nIdx+1); + } /* Begin generating code. */ v = sqlite3GetVdbe(pParse); @@ -374,9 +385,15 @@ void sqlite3Update( sqlite3VdbeSetP4KeyInfo(pParse, pPk); } - /* Begin the database scan. */ + /* Begin the database scan. + ** + ** Do not consider a single-pass strategy for a multi-row update if + ** there are any triggers or foreign keys to process, or rows may + ** be deleted as a result of REPLACE conflict handling. Any of these + ** things might disturb a cursor being used to scan through the table + ** or index, causing a single-pass approach to malfunction. */ flags = WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE; - if( pParse->nested==0 && pTrigger==0 && hasFK==0 && chngKey==0 ){ + if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ flags |= WHERE_ONEPASS_MULTIROW; } pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur); @@ -436,34 +453,20 @@ void sqlite3Update( labelBreak = sqlite3VdbeMakeLabel(v); if( !isView ){ - int iAddrOnce = 0; - /* - ** Open every index that needs updating. Note that if any - ** index could potentially invoke a REPLACE conflict resolution - ** action, then we need to open all indices because we might need - ** to be deleting some records. - */ - if( onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); - }else{ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); - break; - } - } - } + int addrOnce = 0; + + /* Open every index that needs updating. */ if( eOnePass!=ONEPASS_OFF ){ if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){ - iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); - if( iAddrOnce ) sqlite3VdbeJumpHere(v, iAddrOnce); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); } /* Top of the update loop */ @@ -602,7 +605,6 @@ void sqlite3Update( if( !isView ){ int addr1 = 0; /* Address of jump instruction */ - int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); diff --git a/test/update2.test b/test/update2.test index 8aafb2438b..61ac9792da 100644 --- a/test/update2.test +++ b/test/update2.test @@ -84,14 +84,20 @@ do_execsql_test 3.1 { #------------------------------------------------------------------------- # foreach {tn sql} { - 1 { CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c) } - 2 { CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + 1 { + CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE c1(a INTEGER PRIMARY KEY, b, c, d) + } + 2 { + CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE c1(a INT PRIMARY KEY, b, c, d) WITHOUT ROWID; + } } { - execsql { DROP TABLE IF EXISTS b1 } + execsql { DROP TABLE IF EXISTS b1; DROP TABLE IF EXISTS c1; } execsql $sql + do_execsql_test 4.$tn.0 { CREATE UNIQUE INDEX b1c ON b1(c); - INSERT INTO b1 VALUES(1, 'a', 1); INSERT INTO b1 VALUES(2, 'b', 15); INSERT INTO b1 VALUES(3, 'c', 3); @@ -112,7 +118,63 @@ foreach {tn sql} { 6 f 16 7 g 17 } + + do_execsql_test 4.$tn.2 { + CREATE INDEX c1d ON c1(d, b); + CREATE UNIQUE INDEX c1c ON c1(c, b); + + INSERT INTO c1 VALUES(1, 'a', 1, 1); + INSERT INTO c1 VALUES(2, 'a', 15, 2); + INSERT INTO c1 VALUES(3, 'a', 3, 3); + INSERT INTO c1 VALUES(4, 'a', 4, 4); + INSERT INTO c1 VALUES(5, 'a', 5, 5); + INSERT INTO c1 VALUES(6, 'a', 6, 6); + INSERT INTO c1 VALUES(7, 'a', 7, 7); + } + + do_execsql_test 4.$tn.3 { + UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7; + SELECT * FROM c1 ORDER BY a; + } { + 1 a 1 1 + 3 a 3 3 + 4 a 14 4 + 5 a 15 5 + 6 a 16 6 + 7 a 17 7 + } + + do_execsql_test 4.$tn.4 { PRAGMA integrity_check } ok + + do_execsql_test 4.$tn.5 { + DROP INDEX c1d; + DROP INDEX c1c; + DELETE FROM c1; + + INSERT INTO c1 VALUES(1, 'a', 1, 1); + INSERT INTO c1 VALUES(2, 'a', 15, 2); + INSERT INTO c1 VALUES(3, 'a', 3, 3); + INSERT INTO c1 VALUES(4, 'a', 4, 4); + INSERT INTO c1 VALUES(5, 'a', 5, 5); + INSERT INTO c1 VALUES(6, 'a', 6, 6); + INSERT INTO c1 VALUES(7, 'a', 7, 7); + + CREATE INDEX c1d ON c1(d); + CREATE UNIQUE INDEX c1c ON c1(c); + } + + do_execsql_test 4.$tn.6 { + UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7; + SELECT * FROM c1 ORDER BY a; + } { + 1 a 1 1 + 3 a 3 3 + 4 a 14 4 + 5 a 15 5 + 6 a 16 6 + 7 a 17 7 + } } - finish_test + From e206ea7f480e7d68db9aaa7e3415bef3f66d3be6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Jan 2017 20:10:30 +0000 Subject: [PATCH 018/292] Fix a problem causing the pre-update hook to be passed an incorrect rowid value in some single-pass multi-row updates. FossilOrigin-Name: 62257eb53c13d4c7ed128d5d89f6f10d4aff945c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/update.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 95612d61fb..dc4ff7c1a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\ssingle-pass\smulti-row\sUPDATE\sstatements\sthat\sinvoke\sREPLACE\nconflict\shandling. -D 2017-01-11T19:03:08.308 +C Fix\sa\sproblem\scausing\sthe\spre-update\shook\sto\sbe\spassed\san\sincorrect\srowid\nvalue\sin\ssome\ssingle-pass\smulti-row\supdates. +D 2017-01-11T20:10:30.712 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -451,7 +451,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 4ed0fcccccd6488ee8a3bc02df10979b503f25d7 +F src/update.c 715bbfe276c9d008482cdb0ecd86ff996dba3b6c F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cab86c90945126c24c40cf2dedd053a8c46d00d6 -R 91b60fc45aced248f038aee86694df0e +P 0a2b8e1b9dc600b5a93622e8eea6218649df5e0f +R d791ce37848d763756ad4c9c28aa682a U dan -Z f69bdc261645bb06f02a001c239441ef +Z 064d431f5a23ac6118a97592b16afa49 diff --git a/manifest.uuid b/manifest.uuid index c670eb3052..19077faa6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a2b8e1b9dc600b5a93622e8eea6218649df5e0f \ No newline at end of file +62257eb53c13d4c7ed128d5d89f6f10d4aff945c \ No newline at end of file diff --git a/src/update.c b/src/update.c index 242d36b739..e1cbc21da2 100644 --- a/src/update.c +++ b/src/update.c @@ -643,6 +643,10 @@ void sqlite3Update( OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP), regNewRowid ); + if( eOnePass==ONEPASS_MULTI ){ + assert( hasFK==0 && chngKey==0 ); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } From ea8f0a155e64ec6ea300ed5c77a96397e77b0edb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 11:50:08 +0000 Subject: [PATCH 019/292] Make sure Tcl_AppendResult() always has a NULL-pointer argument at the end. FossilOrigin-Name: c07aef6f909fe35de110f0b180dbf5aa4c226af3 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/tclsqlite.c | 8 ++++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 62ade5afe1..7f69b73c9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sallow\ssome\smulti-row\sUPDATE\sstatements\sto\savoid\sthe\stwo-pass\napproach. -D 2017-01-11T21:03:53.460 +C Make\ssure\sTcl_AppendResult()\salways\shas\sa\sNULL-pointer\sargument\sat\sthe\send. +D 2017-01-12T11:50:08.746 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -397,7 +397,7 @@ F src/sqliteInt.h bec6274d8991528bc12d9a34d01fe84bdf6d00d9 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 -F src/tclsqlite.c 205c66b9b81d97978a155caa3ef5be9c4de2b174 +F src/tclsqlite.c 418f5e5e0840425a7e5b33f3600dccd378a57549 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee @@ -1544,8 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f58f75b5a06f88ba97bd1a02bee621c64691c6f8 62257eb53c13d4c7ed128d5d89f6f10d4aff945c -R a5d3781c66cde36056f7269332c03001 -T +closed 62257eb53c13d4c7ed128d5d89f6f10d4aff945c -U dan -Z 8a4ce767e7a0e27eb39847a70d667c24 +P 7ae6104a3e0d1d2cacfe2be732f0220a53908132 +R c90de156797531d5a59462fa9fb70f0d +U drh +Z 140d8d1b5e4e9e613293d0d1ce895be9 diff --git a/manifest.uuid b/manifest.uuid index 5a73a263e8..061e989144 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ae6104a3e0d1d2cacfe2be732f0220a53908132 \ No newline at end of file +c07aef6f909fe35de110f0b180dbf5aa4c226af3 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 1d87c9b331..5b52bf0c91 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2307,7 +2307,7 @@ static int SQLITE_TCLAPI DbObjCmd( } in = fopen(zFile, "rb"); if( in==0 ){ - Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, NULL); + Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0); sqlite3_finalize(pStmt); return TCL_ERROR; } @@ -2536,7 +2536,7 @@ static int SQLITE_TCLAPI DbObjCmd( int n = strlen30(z); if( n>2 && strncmp(z, "-argcount",n)==0 ){ if( i==(objc-2) ){ - Tcl_AppendResult(interp, "option requires an argument: ", z, 0); + Tcl_AppendResult(interp, "option requires an argument: ", z,(char*)0); return TCL_ERROR; } if( Tcl_GetIntFromObj(interp, objv[i+1], &nArg) ) return TCL_ERROR; @@ -2551,7 +2551,7 @@ static int SQLITE_TCLAPI DbObjCmd( flags |= SQLITE_DETERMINISTIC; }else{ Tcl_AppendResult(interp, "bad option \"", z, - "\": must be -argcount or -deterministic", 0 + "\": must be -argcount or -deterministic", (char*)0 ); return TCL_ERROR; } @@ -3208,7 +3208,7 @@ static int SQLITE_TCLAPI DbObjCmd( pObj = Tcl_NewStringObj((char*)sqlite3_value_text(pValue), -1); Tcl_SetObjResult(interp, pObj); }else{ - Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), 0); + Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0); return TCL_ERROR; } } From 24be549329cda7f0628fc119742b649f303027c2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 11:52:47 +0000 Subject: [PATCH 020/292] Remove invalid test case from cursorhints2.test FossilOrigin-Name: 163cc1b2f7100827d5803b81dcd97a1417744f66 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/cursorhint2.test | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f69b73c9c..ecc1cabab4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sTcl_AppendResult()\salways\shas\sa\sNULL-pointer\sargument\sat\sthe\send. -D 2017-01-12T11:50:08.746 +C Remove\sinvalid\stest\scase\sfrom\scursorhints2.test +D 2017-01-12T11:52:47.951 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -626,7 +626,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 -F test/cursorhint2.test fa41f0d997e67db921d08c31e73111b32811201a +F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc F test/dbfuzz.c 8cc2bdb818b4483a052f9f80f96be74cbd9a6e1d F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ae6104a3e0d1d2cacfe2be732f0220a53908132 -R c90de156797531d5a59462fa9fb70f0d +P c07aef6f909fe35de110f0b180dbf5aa4c226af3 +R c1c1751767bcc81e2bffd9195a0f35dd U drh -Z 140d8d1b5e4e9e613293d0d1ce895be9 +Z 744c5ff27f4cf888c7a2ae696c740d23 diff --git a/manifest.uuid b/manifest.uuid index 061e989144..daa470f6ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c07aef6f909fe35de110f0b180dbf5aa4c226af3 \ No newline at end of file +163cc1b2f7100827d5803b81dcd97a1417744f66 \ No newline at end of file diff --git a/test/cursorhint2.test b/test/cursorhint2.test index bc447d0f22..616235b376 100644 --- a/test/cursorhint2.test +++ b/test/cursorhint2.test @@ -91,10 +91,12 @@ do_extract_hints_test 1.6 { t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)} } -do_extract_hints_test 1.7 { - SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f); -} { - t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))} +if 0 { + do_extract_hints_test 1.7 { + SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f); + } { + t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))} + } } #------------------------------------------------------------------------- From b701c9a6c3bf9efef49e0a4dfa9905de9a25a06e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 15:11:03 +0000 Subject: [PATCH 021/292] Improved detection of cells that extend into the reserved space at the end of the page while adjusting overflow page pointers during autovacuum. FossilOrigin-Name: 8097712c9c1f4ea16bc5dd462da248ef98896061 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ecc1cabab4..64318a45bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sinvalid\stest\scase\sfrom\scursorhints2.test -D 2017-01-12T11:52:47.951 +C Improved\sdetection\sof\scells\sthat\sextend\sinto\sthe\sreserved\sspace\sat\sthe\send\nof\sthe\spage\swhile\sadjusting\soverflow\spage\spointers\sduring\sautovacuum. +D 2017-01-12T15:11:03.724 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -331,7 +331,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 44e9612965f63bef288673b81faa43e765bcac5f +F src/btree.c 90f8b15ec9baf65d5bfff65bdaee773b928836c1 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c07aef6f909fe35de110f0b180dbf5aa4c226af3 -R c1c1751767bcc81e2bffd9195a0f35dd +P 163cc1b2f7100827d5803b81dcd97a1417744f66 +R d095162836679cc779e9b3972f22e3ea U drh -Z 744c5ff27f4cf888c7a2ae696c740d23 +Z e05f25d6fff0c473a188620363f2fa54 diff --git a/manifest.uuid b/manifest.uuid index daa470f6ca..ae5d12009d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -163cc1b2f7100827d5803b81dcd97a1417744f66 \ No newline at end of file +8097712c9c1f4ea16bc5dd462da248ef98896061 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 60f886bf5a..99e9741b6f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3356,12 +3356,14 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; pPage->xParseCell(pPage, pCell, &info); - if( info.nLocalaData+pPage->maskPage - && iFrom==get4byte(pCell+info.nSize-4) - ){ - put4byte(pCell+info.nSize-4, iTo); - break; + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } } }else{ if( get4byte(pCell)==iFrom ){ From 70d90a2c37482a9b8a5566ef47bc4101f5073184 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 16:14:33 +0000 Subject: [PATCH 022/292] Remove an unnecessary corruption test from the btree balancer. If corruption is present, it will be found harmlessly by later tests. FossilOrigin-Name: bddf39562d08e259c43dd59b82afb62fe0eb2eef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 64318a45bb..338a7ee537 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sof\scells\sthat\sextend\sinto\sthe\sreserved\sspace\sat\sthe\send\nof\sthe\spage\swhile\sadjusting\soverflow\spage\spointers\sduring\sautovacuum. -D 2017-01-12T15:11:03.724 +C Remove\san\sunnecessary\scorruption\stest\sfrom\sthe\sbtree\sbalancer.\s\sIf\scorruption\nis\spresent,\sit\swill\sbe\sfound\sharmlessly\sby\slater\stests. +D 2017-01-12T16:14:33.708 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -331,7 +331,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 90f8b15ec9baf65d5bfff65bdaee773b928836c1 +F src/btree.c 5f8642134a2b56d0909f6196d7a1b0731cf4188a F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 163cc1b2f7100827d5803b81dcd97a1417744f66 -R d095162836679cc779e9b3972f22e3ea +P 8097712c9c1f4ea16bc5dd462da248ef98896061 +R c3fb4d0a8856fd8a9a7ac25a5dcf6c2d U drh -Z e05f25d6fff0c473a188620363f2fa54 +Z 126b06a4044b551a39fa62a47a4fd8cf diff --git a/manifest.uuid b/manifest.uuid index ae5d12009d..0dbbd81794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8097712c9c1f4ea16bc5dd462da248ef98896061 \ No newline at end of file +bddf39562d08e259c43dd59b82afb62fe0eb2eef \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 99e9741b6f..c635e15718 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7271,7 +7271,6 @@ static int balance_nonroot( for(i=0; inFree; - if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); } From ae051a8970d0f50869fb8a36a5d51a13a8c929c0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 16:21:54 +0000 Subject: [PATCH 023/292] Fix harmless compiler warnings in the UPDATE code generator. FossilOrigin-Name: 385db266673abaf7013ffad09b28014c246547ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/update.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 338a7ee537..1e60a98460 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\scorruption\stest\sfrom\sthe\sbtree\sbalancer.\s\sIf\scorruption\nis\spresent,\sit\swill\sbe\sfound\sharmlessly\sby\slater\stests. -D 2017-01-12T16:14:33.708 +C Fix\sharmless\scompiler\swarnings\sin\sthe\sUPDATE\scode\sgenerator. +D 2017-01-12T16:21:54.480 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -451,7 +451,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 715bbfe276c9d008482cdb0ecd86ff996dba3b6c +F src/update.c 4e21634dde80a59b85f4d59eca75e3b5a5001fd4 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8097712c9c1f4ea16bc5dd462da248ef98896061 -R c3fb4d0a8856fd8a9a7ac25a5dcf6c2d +P bddf39562d08e259c43dd59b82afb62fe0eb2eef +R c1f97b0b1d9fb4e22bf29bd06fe30297 U drh -Z 126b06a4044b551a39fa62a47a4fd8cf +Z a2b5fe338be7534e4379cc3897438f5d diff --git a/manifest.uuid b/manifest.uuid index 0dbbd81794..da8dc7b7b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bddf39562d08e259c43dd59b82afb62fe0eb2eef \ No newline at end of file +385db266673abaf7013ffad09b28014c246547ef \ No newline at end of file diff --git a/src/update.c b/src/update.c index e1cbc21da2..8bdb740c16 100644 --- a/src/update.c +++ b/src/update.c @@ -132,9 +132,9 @@ void sqlite3Update( int iEph = 0; /* Ephemeral table holding all primary key values */ int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ - int addrOpen; /* Address of OP_OpenEphemeral */ - int iPk; /* First of nPk cells holding PRIMARY KEY value */ - i16 nPk; /* Number of components of the PRIMARY KEY */ + int addrOpen = 0; /* Address of OP_OpenEphemeral */ + int iPk = 0; /* First of nPk cells holding PRIMARY KEY value */ + i16 nPk = 0; /* Number of components of the PRIMARY KEY */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Register Allocations */ From 7b20a15d1380ed7b2415e3b489695dbcd016c039 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Jan 2017 19:10:55 +0000 Subject: [PATCH 024/292] Remove a branch that is probably unreachable, and which adds no value. FossilOrigin-Name: 9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1e60a98460..000de47de4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\sUPDATE\scode\sgenerator. -D 2017-01-12T16:21:54.480 +C Remove\sa\sbranch\sthat\sis\sprobably\sunreachable,\sand\swhich\sadds\sno\svalue. +D 2017-01-12T19:10:55.750 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -331,7 +331,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 5f8642134a2b56d0909f6196d7a1b0731cf4188a +F src/btree.c 69966fb2c574954cd3216f09407c9a02c52d3bd7 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bddf39562d08e259c43dd59b82afb62fe0eb2eef -R c1f97b0b1d9fb4e22bf29bd06fe30297 +P 385db266673abaf7013ffad09b28014c246547ef +R fc07cbd5c2e564fa0b0693e0166fd1a4 U drh -Z a2b5fe338be7534e4379cc3897438f5d +Z 1662f62ac3b6c5068ca23b30e30b82dc diff --git a/manifest.uuid b/manifest.uuid index da8dc7b7b9..644e2b84c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -385db266673abaf7013ffad09b28014c246547ef \ No newline at end of file +9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c635e15718..7b56867646 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8124,7 +8124,7 @@ int sqlite3BtreeInsert( pCur->eState = CURSOR_INVALID; if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ rc = moveToRoot(pCur); - if( pCur->pKeyInfo && rc==SQLITE_OK ){ + if( pCur->pKeyInfo ){ assert( pCur->pKey==0 ); pCur->pKey = sqlite3Malloc( pX->nKey ); if( pCur->pKey==0 ){ From be7a0cee4eb3be876182a23f004b4de2d8f2103f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Jan 2017 12:53:35 +0000 Subject: [PATCH 025/292] Fix the build for SQLITE_ENABLE_MEMORY_MANAGEMENT. FossilOrigin-Name: 8c85b8fdd7f0ba65fba83361d361a567b797a184 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 000de47de4..21c132777c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sbranch\sthat\sis\sprobably\sunreachable,\sand\swhich\sadds\sno\svalue. -D 2017-01-12T19:10:55.750 +C Fix\sthe\sbuild\sfor\sSQLITE_ENABLE_MEMORY_MANAGEMENT. +D 2017-01-13T12:53:35.623 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -354,7 +354,7 @@ F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c c36ef8fa6e4cc53ec258c5aa3ad86eb47350cc3d +F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1544,7 +1544,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 385db266673abaf7013ffad09b28014c246547ef -R fc07cbd5c2e564fa0b0693e0166fd1a4 +P 9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4 +R 538795cf3f379197890291f480545c18 U drh -Z 1662f62ac3b6c5068ca23b30e30b82dc +Z 8d3190b0c41cbb057e375dc759a5947e diff --git a/manifest.uuid b/manifest.uuid index 644e2b84c1..a786fc5f1c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4 \ No newline at end of file +8c85b8fdd7f0ba65fba83361d361a567b797a184 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 053b57b47f..0a3d891e9b 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -219,11 +219,12 @@ static void sqlite3MallocAlarm(int nByte){ */ static void mallocWithAlarm(int n, void **pp){ void *p; + int nFull = 0; assert( sqlite3_mutex_held(mem0.mutex) ); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - int nFull = sqlite3GlobalConfig.m.xRoundup(n); + nFull = sqlite3GlobalConfig.m.xRoundup(n); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -239,7 +240,7 @@ static void mallocWithAlarm(int n, void **pp){ } #endif if( p ){ - int nFull = sqlite3MallocSize(p); + nFull = sqlite3MallocSize(p); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } From ef2f5925d8b4f9862110c5bea0c18fe131608974 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 13 Jan 2017 18:24:37 +0000 Subject: [PATCH 026/292] Fix a problem preventing resumption of RBU operations after recovering from a process or system failure that occurs during the incremental-checkpoint phase. FossilOrigin-Name: 97914266cb4ec63b0c9185ab139673139bd2f0ed --- ext/rbu/rburesume.test | 254 +++++++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.c | 2 +- manifest | 15 +-- manifest.uuid | 2 +- 4 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 ext/rbu/rburesume.test diff --git a/ext/rbu/rburesume.test b/ext/rbu/rburesume.test new file mode 100644 index 0000000000..d03894e135 --- /dev/null +++ b/ext/rbu/rburesume.test @@ -0,0 +1,254 @@ +# 2017 January 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for resumption of RBU operations in the +# case where the previous RBU process crashed. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rburesume + +forcedelete test.db-shm test.db-oal +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s; +} +db_save_and_close + +do_test 1.1 { + list [file exists test.db] \ + [file exists test.db-wal] \ + [file exists test.db-shm] \ + [file exists test.db-oal] +} {1 0 0 0} + +# Each iteration of the following loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Closes the RBU vacuum handled opened in (2). +# 4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + rbu close + if {$rc == "SQLITE_DONE"} break + + do_test 1.2.$n.1 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.2.$n.2 { + sqlite3 db2 test.db + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.3.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.3.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it 10 times. Then closes it. +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (3) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<10} {incr i} { + rbu step + } + rbu close + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.4.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.4.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +forcedelete rbu.db +do_test 2.0 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO data_t1 + SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s; + } + db2 close +} {} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU handle to apply the RBU update created in test case 2.0. +# 3. Steps the RBU handle $n times. +# 4. Takes a copy of all database files and the state db. +# 5. Opens a second RBU handle on the copy, resumes and completes the +# RBU op. Checks it worked as expected. +# +# The loop runs until $n is large enough that step (3) applies the entire +# update. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu rbu test.db rbu.db state.db + + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 2.$n.1 { + sqlite3rbu rbu test.db2 rbu.db state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 2.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {60 ok} + db2 close +} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 746469a8af..235ba3fe2a 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2408,7 +2408,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ }else{ RbuState *pState = rbuLoadState(p); if( pState ){ - bOpen = (pState->eStage>RBU_STAGE_MOVE); + bOpen = (pState->eStage>=RBU_STAGE_MOVE); rbuFreeState(pState); } } diff --git a/manifest b/manifest index 21c132777c..d943af7804 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sfor\sSQLITE_ENABLE_MEMORY_MANAGEMENT. -D 2017-01-13T12:53:35.623 +C Fix\sa\sproblem\spreventing\sresumption\sof\sRBU\soperations\safter\srecovering\sfrom\sa\nprocess\sor\ssystem\sfailure\sthat\soccurs\sduring\sthe\sincremental-checkpoint\sphase. +D 2017-01-13T18:24:37.297 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -254,10 +254,11 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b +F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc -F ext/rbu/sqlite3rbu.c e074c38798b90591f7f0cf0032d62f152ce5a95e +F ext/rbu/sqlite3rbu.c 4ca89fbe9ca501da17f338d580d3f19d59fc6053 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1544,7 +1545,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4 -R 538795cf3f379197890291f480545c18 -U drh -Z 8d3190b0c41cbb057e375dc759a5947e +P 8c85b8fdd7f0ba65fba83361d361a567b797a184 +R dc62936e86267740ae9736f88160b05f +U dan +Z f2366ce5ecd39574b5b5eb81b3b99a63 diff --git a/manifest.uuid b/manifest.uuid index a786fc5f1c..e44a5e6607 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c85b8fdd7f0ba65fba83361d361a567b797a184 \ No newline at end of file +97914266cb4ec63b0c9185ab139673139bd2f0ed \ No newline at end of file From ce31643fd9d7989a90a6cd73d8b5dd4896e323eb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 16 Jan 2017 16:01:50 +0000 Subject: [PATCH 027/292] Add test cases for tickets [91e2e8ba6ff2e2] and [7ffd1ca1d2ad4ec]. FossilOrigin-Name: 9d0dfe0b088a5917afa06207ca3ac5618e3da82f --- manifest | 16 +++++--- manifest.uuid | 2 +- test/affinity3.test | 91 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 test/affinity3.test diff --git a/manifest b/manifest index d943af7804..9eb1084776 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sresumption\sof\sRBU\soperations\safter\srecovering\sfrom\sa\nprocess\sor\ssystem\sfailure\sthat\soccurs\sduring\sthe\sincremental-checkpoint\sphase. -D 2017-01-13T18:24:37.297 +C Add\stest\scases\sfor\stickets\s[91e2e8ba6ff2e2]\sand\s[7ffd1ca1d2ad4ec]. +D 2017-01-16T16:01:50.379 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -476,6 +476,7 @@ F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 F src/whereexpr.c 35ad025389a632a3987a35617c878be3b3d70dc6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd +F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -1545,7 +1546,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c85b8fdd7f0ba65fba83361d361a567b797a184 -R dc62936e86267740ae9736f88160b05f -U dan -Z f2366ce5ecd39574b5b5eb81b3b99a63 +P 97914266cb4ec63b0c9185ab139673139bd2f0ed +R b7007367e54a15c0816f3ba9a91a394f +T *branch * automatic-index-affinity +T *sym-automatic-index-affinity * +T -sym-trunk * +U drh +Z 2b3d4a827097e1fdb9a913f5a46e57c3 diff --git a/manifest.uuid b/manifest.uuid index e44a5e6607..48c1698079 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97914266cb4ec63b0c9185ab139673139bd2f0ed \ No newline at end of file +9d0dfe0b088a5917afa06207ca3ac5618e3da82f \ No newline at end of file diff --git a/test/affinity3.test b/test/affinity3.test new file mode 100644 index 0000000000..a335618a74 --- /dev/null +++ b/test/affinity3.test @@ -0,0 +1,91 @@ +# 2017-01-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for bugs: +# +# https://www.sqlite.org/src/info/91e2e8ba6ff2e2 +# https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Ticket https://www.sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19) +# Automatic index causes undesired type conversions +# +do_execsql_test affinity3-100 { + CREATE TABLE customer (id INT PRIMARY KEY); + CREATE TABLE apr (id INT PRIMARY KEY, apr REAL); + + CREATE VIEW v1 AS + SELECT c.id, i.apr + FROM customer c + LEFT JOIN apr i ON i.id=c.id; + + CREATE VIEW v2 AS + SELECT c.id, v1.apr + FROM customer c + LEFT JOIN v1 ON v1.id=c.id; + + INSERT INTO customer (id) VALUES (1); + INSERT INTO apr (id, apr) VALUES (1, 12); + INSERT INTO customer (id) VALUES (2); + INSERT INTO apr (id, apr) VALUES (2, 12.01); +} +do_execsql_test affinity3-110 { + PRAGMA automatic_index=ON; + SELECT id, (apr / 100), typeof(apr) apr_type FROM v1; +} {1 0.12 real 2 0.1201 real} +do_execsql_test affinity3-120 { + SELECT id, (apr / 100), typeof(apr) apr_type FROM v2; +} {1 0.12 real 2 0.1201 real} +do_execsql_test affinity3-130 { + PRAGMA automatic_index=OFF; + SELECT id, (apr / 100), typeof(apr) apr_type FROM v1; +} {1 0.12 real 2 0.1201 real} +do_execsql_test affinity3-140 { + SELECT id, (apr / 100), typeof(apr) apr_type FROM v2; +} {1 0.12 real 2 0.1201 real} + +# Ticket https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf (2017-01-16) +# Incorrect affinity when using automatic indexes +# +do_execsql_test affinity3-200 { + CREATE TABLE map_integer (id INT, name); + INSERT INTO map_integer VALUES(1,'a'); + CREATE TABLE map_text (id TEXT, name); + INSERT INTO map_text VALUES('4','e'); + CREATE TABLE data (id TEXT, name); + INSERT INTO data VALUES(1,'abc'); + INSERT INTO data VALUES('4','xyz'); + CREATE VIEW idmap as + SELECT * FROM map_integer + UNION SELECT * FROM map_text; + CREATE TABLE mzed AS SELECT * FROM idmap; +} + +do_execsql_test affinity3-210 { + PRAGMA automatic_index=ON; + SELECT * FROM data JOIN idmap USING(id); +} {1 abc a 4 xyz e} +do_execsql_test affinity3-220 { + SELECT * FROM data JOIN mzed USING(id); +} {1 abc a 4 xyz e} + +do_execsql_test affinity3-250 { + PRAGMA automatic_index=OFF; + SELECT * FROM data JOIN idmap USING(id); +} {1 abc a 4 xyz e} +do_execsql_test affinity3-260 { + SELECT * FROM data JOIN mzed USING(id); +} {1 abc a 4 xyz e} + +finish_test From da060052e3d6436e5998f94bb7c92ce1a70d5004 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 16 Jan 2017 16:43:02 +0000 Subject: [PATCH 028/292] Back out check-in [0b3174e0b1364c] and replace it with a better fix for ticket [91e2e8ba6ff2e2] - a fix that does not cause the problem identified by ticket [7ffd1ca1d2ad4ec]. FossilOrigin-Name: 0613665274346917f5482f9210bf0c60a0fed7d9 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/delete.c | 4 ++++ src/update.c | 10 +++++----- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 9eb1084776..c9e37e2c9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\stickets\s[91e2e8ba6ff2e2]\sand\s[7ffd1ca1d2ad4ec]. -D 2017-01-16T16:01:50.379 +C Back\sout\scheck-in\s[0b3174e0b1364c]\sand\sreplace\sit\swith\sa\sbetter\sfix\nfor\sticket\s[91e2e8ba6ff2e2]\s-\sa\sfix\sthat\sdoes\snot\scause\sthe\sproblem\nidentified\sby\sticket\s[7ffd1ca1d2ad4ec]. +D 2017-01-16T16:43:02.076 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -341,7 +341,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c a84f6229ccb9448460c287248024ceb70e10baab +F src/delete.c 8a444fea8340989d6b1be2e122c55bfc61ce69be F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae @@ -452,7 +452,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c 4e21634dde80a59b85f4d59eca75e3b5a5001fd4 +F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -1546,10 +1546,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 97914266cb4ec63b0c9185ab139673139bd2f0ed -R b7007367e54a15c0816f3ba9a91a394f -T *branch * automatic-index-affinity -T *sym-automatic-index-affinity * -T -sym-trunk * +P 9d0dfe0b088a5917afa06207ca3ac5618e3da82f +R d9bec5fa752ad661e71dee0a4ee8e911 U drh -Z 2b3d4a827097e1fdb9a913f5a46e57c3 +Z 5e9d81347f66aedf2ffc7c8c671d9f61 diff --git a/manifest.uuid b/manifest.uuid index 48c1698079..d656223073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d0dfe0b088a5917afa06207ca3ac5618e3da82f \ No newline at end of file +0613665274346917f5482f9210bf0c60a0fed7d9 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 1a5edb59d8..1d31622237 100644 --- a/src/delete.c +++ b/src/delete.c @@ -873,6 +873,10 @@ int sqlite3GenerateIndexKey( } if( regOut ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + if( pIdx->pTable->pSelect ){ + const char *zAff = sqlite3IndexAffinityStr(pParse->db, pIdx); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); + } } sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; diff --git a/src/update.c b/src/update.c index 8bdb740c16..45af001c3e 100644 --- a/src/update.c +++ b/src/update.c @@ -71,12 +71,12 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ if( pValue ){ sqlite3VdbeAppendP4(v, pValue, P4_MEM); } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } -#endif } +#ifndef SQLITE_OMIT_FLOATING_POINT + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif } /* From c711f53f30a5c1c7017deb2a77a18d806c936fba Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Jan 2017 00:10:58 +0000 Subject: [PATCH 029/292] Disable intrinsic functions for Windows using Clang, due to reports of linkage errors. This causes a 0.6% performance reduction. We will want to revisit this change in the future. FossilOrigin-Name: 7fd560c6d2ff470b755ad118287a0a8825b3009e --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a9e3f89018..2715f1f973 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\scheck-in\s[0b3174e0b1364c]\sand\sreplace\sit\swith\sa\sbetter\sfix\sfor\s\\ticket\s[91e2e8ba6ff2e2]\s-\sa\sfix\sthat\sdoes\snot\scause\sthe\sproblem\sidentified\sby\s\nticket\s[7ffd1ca1d2ad4ec].\sAdd\snew\stest\scases\sfor\sboth\stickets. -D 2017-01-16T18:10:17.967 +C Disable\sintrinsic\sfunctions\sfor\sWindows\susing\sClang,\sdue\sto\sreports\sof\nlinkage\serrors.\s\sThis\scauses\sa\s0.6%\sperformance\sreduction.\s\sWe\swill\swant\sto\nrevisit\sthis\schange\sin\sthe\sfuture. +D 2017-01-17T00:10:58.166 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -394,7 +394,7 @@ F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h bec6274d8991528bc12d9a34d01fe84bdf6d00d9 +F src/sqliteInt.h ce3e07c720b0cebc8887ea86b3b128da0913c5d3 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -1546,8 +1546,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 97914266cb4ec63b0c9185ab139673139bd2f0ed 0613665274346917f5482f9210bf0c60a0fed7d9 -R d9bec5fa752ad661e71dee0a4ee8e911 -T +closed 0613665274346917f5482f9210bf0c60a0fed7d9 +P 9b64af7b5201a8700ae9e384b04714ca18df7449 +R 2a65551e28553413c063bc022e23faf5 U drh -Z 535f9cb18662b936eb37d1700bf0881b +Z 6f331e0f247acb77a734fb284e31d3c4 diff --git a/manifest.uuid b/manifest.uuid index 970b95dbeb..30780fc11a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b64af7b5201a8700ae9e384b04714ca18df7449 \ No newline at end of file +7fd560c6d2ff470b755ad118287a0a8825b3009e \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3215a7f865..d5ac957d22 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -111,7 +111,7 @@ #endif /* What version of CLANG is being used. 0 means CLANG is not being used */ -#ifdef __clang__ +#if defined(__clang__) && !defined(_WIN32) # define CLANG_VERSION \ (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) #else From 25fd2e247bca3d0ddf9a88260bcea7ddfac268bb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 17 Jan 2017 10:41:42 +0000 Subject: [PATCH 030/292] Fix a problem that could cause a spurious SQLITE_NOMEM error when attempting to resume an RBU operation if the previous client failed right after completing the incremental checkpoint. Also a "cannot vacuum wal db" error that could occur when resuming an RBU vacuum if an error (OOM or IO error) occurs during the incremental checkpoint. FossilOrigin-Name: 681d96eb822e606da53700867191d4738bda20c8 --- ext/rbu/rbufault4.test | 66 +++++++++++++++++++++++++++++++++++++++++ ext/rbu/rbuvacuum2.test | 31 +++++++++++++++++++ ext/rbu/sqlite3rbu.c | 37 ++++++++++++++++++----- manifest | 17 ++++++----- manifest.uuid | 2 +- 5 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 ext/rbu/rbufault4.test diff --git a/ext/rbu/rbufault4.test b/ext/rbu/rbufault4.test new file mode 100644 index 0000000000..fe6cc90dc9 --- /dev/null +++ b/ext/rbu/rbufault4.test @@ -0,0 +1,66 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix rbufault4 + +for {set tn 1} {1} {incr tn} { + reset_db + do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + for {set i 0} {$i < $tn} {incr i} { rbu step } + set rc [rbu close] + if {$rc!="SQLITE_OK"} { + if {$rc!="SQLITE_DONE"} {error $rc} + break + } + faultsim_save + + do_faultsim_test $tn -faults oom-t* -prep { + faultsim_restore + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while 1 { + set rc [rbu step] + if {$rc=="SQLITE_DONE"} break + if {$rc!="SQLITE_OK"} { error $rc } + } + } -test { + catch {rbu close} + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} + + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + set trc [rbu close] + if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" } + + set rc [db one {PRAGMA integrity_check}] + if {$rc!="ok"} { error "Got $rc instead of ok!" } + } +} + + + +finish_test + diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test index bd38660320..0a1fe3da94 100644 --- a/ext/rbu/rbuvacuum2.test +++ b/ext/rbu/rbuvacuum2.test @@ -199,6 +199,37 @@ if {$::tcl_platform(platform)=="unix"} { } } +#------------------------------------------------------------------------- +# Test the outcome of some other connection running a checkpoint while +# the incremental checkpoint is suspended. +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} +forcedelete test.db2 + +do_test 6.1 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu state]!="checkpoint"} { rbu step } + rbu close +} {SQLITE_OK} + +do_execsql_test 6.2 { + SELECT 1 FROM sqlite_master LIMIT 1; + PRAGMA wal_checkpoint; +} {1 0 4 4} + +do_test 6.3 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]!="SQLITE_DONE"} { rbu step } + rbu close + execsql { PRAGMA integrity_check } +} {ok} finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 235ba3fe2a..48c69115ee 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2333,7 +2333,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. */ -static void rbuOpenDatabase(sqlite3rbu *p){ +static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2420,6 +2420,15 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ @@ -2600,16 +2609,18 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && p->nFrame>0 ){ p->eStage = RBU_STAGE_CKPT; p->nStep = (pState ? pState->nRow : 0); p->aBuf = rbuMalloc(p, p->pgsz); p->iWalCksum = rbuShmChecksum(p); } - if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + } } } @@ -2782,7 +2793,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ #endif if( p->rc==SQLITE_OK ){ - rbuOpenDatabase(p); + rbuOpenDatabase(p, 0); rbuSetupCheckpoint(p, 0); } } @@ -3493,6 +3504,7 @@ static sqlite3rbu *openRbuHandle( /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ char *pCsr = (char*)&p[1]; + int bRetry = 0; if( zTarget ){ p->zTarget = pCsr; memcpy(p->zTarget, zTarget, nTarget+1); @@ -3504,7 +3516,18 @@ static sqlite3rbu *openRbuHandle( if( zState ){ p->zState = rbuMPrintf(p, "%s", zState); } - rbuOpenDatabase(p); + + /* If the first attempt to open the database file fails and the bRetry + ** flag it set, this means that the db was not opened because it seemed + ** to be a wal-mode db. But, this may have happened due to an earlier + ** RBU vacuum operation leaving an old wal file in the directory. + ** If this is the case, it will have been checkpointed and deleted + ** when the handle was closed and a second attempt to open the + ** database may succeed. */ + rbuOpenDatabase(p, &bRetry); + if( bRetry ){ + rbuOpenDatabase(p, 0); + } } if( p->rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 2715f1f973..e888f901e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sintrinsic\sfunctions\sfor\sWindows\susing\sClang,\sdue\sto\sreports\sof\nlinkage\serrors.\s\sThis\scauses\sa\s0.6%\sperformance\sreduction.\s\sWe\swill\swant\sto\nrevisit\sthis\schange\sin\sthe\sfuture. -D 2017-01-17T00:10:58.166 +C Fix\sa\sproblem\sthat\scould\scause\sa\sspurious\sSQLITE_NOMEM\serror\swhen\sattempting\nto\sresume\san\sRBU\soperation\sif\sthe\sprevious\sclient\sfailed\sright\safter\ncompleting\sthe\sincremental\scheckpoint.\sAlso\sa\s"cannot\svacuum\swal\sdb"\serror\nthat\scould\soccur\swhen\sresuming\san\sRBU\svacuum\sif\san\serror\s(OOM\sor\sIO\serror)\noccurs\sduring\sthe\sincremental\scheckpoint. +D 2017-01-17T10:41:42.780 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -252,13 +252,14 @@ F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca +F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 -F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc -F ext/rbu/sqlite3rbu.c 4ca89fbe9ca501da17f338d580d3f19d59fc6053 +F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa +F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1546,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b64af7b5201a8700ae9e384b04714ca18df7449 -R 2a65551e28553413c063bc022e23faf5 -U drh -Z 6f331e0f247acb77a734fb284e31d3c4 +P 7fd560c6d2ff470b755ad118287a0a8825b3009e +R 3b5f909693e02732236a5ff9d24ebb43 +U dan +Z 623656530433c4c90deab7fe9f3d0d79 diff --git a/manifest.uuid b/manifest.uuid index 30780fc11a..4ce4080714 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fd560c6d2ff470b755ad118287a0a8825b3009e \ No newline at end of file +681d96eb822e606da53700867191d4738bda20c8 \ No newline at end of file From 8a9e83583bc378043e7ba677bb5f40af4614d641 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 18 Jan 2017 22:16:20 +0000 Subject: [PATCH 031/292] Fix handling of initial hidden and/or system files in the opendir() implementation for Windows. No changes to non-test code. FossilOrigin-Name: 26dd42b462dc621b8b0a2295fc91d3e61ac732b6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_windirent.c | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e888f901e2..4727a38a08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\scould\scause\sa\sspurious\sSQLITE_NOMEM\serror\swhen\sattempting\nto\sresume\san\sRBU\soperation\sif\sthe\sprevious\sclient\sfailed\sright\safter\ncompleting\sthe\sincremental\scheckpoint.\sAlso\sa\s"cannot\svacuum\swal\sdb"\serror\nthat\scould\soccur\swhen\sresuming\san\sRBU\svacuum\sif\san\serror\s(OOM\sor\sIO\serror)\noccurs\sduring\sthe\sincremental\scheckpoint. -D 2017-01-17T10:41:42.780 +C Fix\shandling\sof\sinitial\shidden\sand/or\ssystem\sfiles\sin\sthe\sopendir()\simplementation\sfor\sWindows.\s\sNo\schanges\sto\snon-test\scode. +D 2017-01-18T22:16:20.672 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -446,7 +446,7 @@ F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c 600398db0198ca1c77ca183831bf456746b6f5c4 +F src/test_windirent.c 5234b0c38bda5cb7dfc031db6a594af2cbcf7fb7 F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7fd560c6d2ff470b755ad118287a0a8825b3009e -R 3b5f909693e02732236a5ff9d24ebb43 -U dan -Z 623656530433c4c90deab7fe9f3d0d79 +P 681d96eb822e606da53700867191d4738bda20c8 +R 728f11ac297b564a808bf2eb4a24c6c8 +U mistachkin +Z 724393d72bdc7af5ff5eea47dd7cb77d diff --git a/manifest.uuid b/manifest.uuid index 4ce4080714..a40c13f3d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -681d96eb822e606da53700867191d4738bda20c8 \ No newline at end of file +26dd42b462dc621b8b0a2295fc91d3e61ac732b6 \ No newline at end of file diff --git a/src/test_windirent.c b/src/test_windirent.c index 044bc7f414..450e4c3cc7 100644 --- a/src/test_windirent.c +++ b/src/test_windirent.c @@ -63,6 +63,7 @@ LPDIR opendir( dirname = windirent_getenv("SystemDrive"); } + memset(&data, 0, sizeof(struct _finddata_t)); _snprintf(data.name, namesize, "%s\\*", dirname); dirp->d_handle = _findfirst(data.name, &data); @@ -71,12 +72,19 @@ LPDIR opendir( return NULL; } - /* TODO: Remove this block to allow hidden and system files. */ + /* TODO: Remove this block to allow hidden and/or system files. */ if( data.attrib&_A_HIDDEN || data.attrib&_A_SYSTEM ){ +next: + + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ closedir(dirp); return NULL; } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( data.attrib&_A_HIDDEN ) goto next; + if( data.attrib&_A_SYSTEM ) goto next; } dirp->d_first.d_attributes = data.attrib; @@ -105,9 +113,10 @@ LPDIRENT readdir( next: + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; - /* TODO: Remove this block to allow hidden and system files. */ + /* TODO: Remove this block to allow hidden and/or system files. */ if( data.attrib&_A_HIDDEN ) goto next; if( data.attrib&_A_SYSTEM ) goto next; @@ -146,12 +155,13 @@ INT readdir_r( next: + memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ *result = NULL; return ENOENT; } - /* TODO: Remove this block to allow hidden and system files. */ + /* TODO: Remove this block to allow hidden and/or system files. */ if( data.attrib&_A_HIDDEN ) goto next; if( data.attrib&_A_SYSTEM ) goto next; From 7059138a0811ea83b98753b632742a4aa71e7c4d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 18 Jan 2017 22:16:34 +0000 Subject: [PATCH 032/292] Make the vtabH-3.1 test more portable and robust. FossilOrigin-Name: d3c91c1fb345fbcbfc60a897bebf771c795430c9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/vtabH.test | 47 +++++++++++++++++++++++++++++++---------------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 4727a38a08..68a0430236 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\sinitial\shidden\sand/or\ssystem\sfiles\sin\sthe\sopendir()\simplementation\sfor\sWindows.\s\sNo\schanges\sto\snon-test\scode. -D 2017-01-18T22:16:20.672 +C Make\sthe\svtabH-3.1\stest\smore\sportable\sand\srobust. +D 2017-01-18T22:16:34.363 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -1383,7 +1383,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 97f61b0253260831af6232163f7852e2653baed6 +F test/vtabH.test a99d22d77e7ad367ea95df7c7c953320980bf63f F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 681d96eb822e606da53700867191d4738bda20c8 -R 728f11ac297b564a808bf2eb4a24c6c8 +P 26dd42b462dc621b8b0a2295fc91d3e61ac732b6 +R ea3d0fbdb25858ca04efea2603f3e30f U mistachkin -Z 724393d72bdc7af5ff5eea47dd7cb77d +Z 79e95e457c2d340f3ec156d2b8c6aa1c diff --git a/manifest.uuid b/manifest.uuid index a40c13f3d9..93490dc381 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -26dd42b462dc621b8b0a2295fc91d3e61ac732b6 \ No newline at end of file +d3c91c1fb345fbcbfc60a897bebf771c795430c9 \ No newline at end of file diff --git a/test/vtabH.test b/test/vtabH.test index f2a116f350..678557385a 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -121,29 +121,43 @@ if {$tcl_platform(platform)!="windows" || \ SELECT name FROM fsdir WHERE dir = '.' AND name = '.' } {test.db .} + proc sort_files { names {nocase false} } { + if {$nocase && $::tcl_platform(platform) eq "windows"} { + return [lsort -dictionary -nocase $names] + } else { + return [lsort $names] + } + } + proc list_root_files {} { if {$::tcl_platform(platform) eq "windows"} { - set res [list] - foreach name [glob -directory $::env(fstreeDrive)/ -- *] { + set res [list]; set dir $::env(fstreeDrive)/; set names [list] + eval lappend names [glob -nocomplain -directory $dir -- *] + foreach name $names { if {[string index [file tail $name] 0] eq "."} continue + if {[file attributes $name -hidden]} continue + if {[file attributes $name -system]} continue lappend res $name } - return $res + return [sort_files $res true] } else { - return [string map {/ {}} [glob /*]] + return [sort_files [string map {/ {}} [glob -nocomplain -- /*]]] } } proc list_files { pattern } { if {$::tcl_platform(platform) eq "windows"} { - set res [list] - foreach name [glob -nocomplain $pattern] { + set res [list]; set names [list] + eval lappend names [glob -nocomplain -- $pattern] + foreach name $names { if {[string index [file tail $name] 0] eq "."} continue + if {[file attributes $name -hidden]} continue + if {[file attributes $name -system]} continue lappend res $name } - return $res + return [sort_files $res] } else { - return [glob -nocomplain $pattern] + return [sort_files [glob -nocomplain -- $pattern]] } } @@ -153,18 +167,19 @@ if {$tcl_platform(platform)!="windows" || \ # set res [list] set root_files [list_root_files] - set num_root_files [llength $root_files] - set lim_root_files [expr {$num_root_files > 5 ? 5 : $num_root_files}] - foreach p [lrange $root_files 0 [expr {$lim_root_files - 1}]] { + foreach p $root_files { if {$::tcl_platform(platform) eq "windows"} { - if {[regexp {\$} $p]} {incr lim_root_files -1} else {lappend res $p} + if {![regexp {\$} $p]} {lappend res $p} } else { lappend res "/$p" } } - do_execsql_test 3.1 [subst { - SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $lim_root_files; - }] $res + set num_root_files [llength $root_files] + do_test 3.1 { + sort_files [execsql { + SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files; + }] true + } [sort_files $res true] # Read all entries in the current directory. # @@ -182,7 +197,7 @@ if {$tcl_platform(platform)!="windows" || \ set res [contents $pwd] do_execsql_test 3.2 { SELECT path FROM fstree WHERE path GLOB $pwd ORDER BY 1 - } [lsort $res] + } [sort_files $res] # Add some sub-directories and files to the current directory. # From d3e73d6c1c3161111d494e84859a640d3357c3e7 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 18 Jan 2017 22:19:01 +0000 Subject: [PATCH 033/292] Remove superfluous option to Tcl 'lsort' in the vtabH test file. FossilOrigin-Name: b92cc6e58ae31cbe6600a522beb5485f7add04b2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/vtabH.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 68a0430236..416ccf1043 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\svtabH-3.1\stest\smore\sportable\sand\srobust. -D 2017-01-18T22:16:34.363 +C Remove\ssuperfluous\soption\sto\sTcl\s'lsort'\sin\sthe\svtabH\stest\sfile. +D 2017-01-18T22:19:01.508 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -1383,7 +1383,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test a99d22d77e7ad367ea95df7c7c953320980bf63f +F test/vtabH.test a2912cd3ea3386aecc3cb74ebfdfcc4e6d4b7dd3 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 26dd42b462dc621b8b0a2295fc91d3e61ac732b6 -R ea3d0fbdb25858ca04efea2603f3e30f +P d3c91c1fb345fbcbfc60a897bebf771c795430c9 +R 2cfcfd69ff656c8eb0c9b18b58b2f78e U mistachkin -Z 79e95e457c2d340f3ec156d2b8c6aa1c +Z e24ce1e7a409b1f16613abdf400a1354 diff --git a/manifest.uuid b/manifest.uuid index 93490dc381..936488379e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3c91c1fb345fbcbfc60a897bebf771c795430c9 \ No newline at end of file +b92cc6e58ae31cbe6600a522beb5485f7add04b2 \ No newline at end of file diff --git a/test/vtabH.test b/test/vtabH.test index 678557385a..3ce457ff0b 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -123,7 +123,7 @@ if {$tcl_platform(platform)!="windows" || \ proc sort_files { names {nocase false} } { if {$nocase && $::tcl_platform(platform) eq "windows"} { - return [lsort -dictionary -nocase $names] + return [lsort -nocase $names] } else { return [lsort $names] } From aac853bae8281e32319c1e5382e2893fa5fb7492 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 18 Jan 2017 22:47:42 +0000 Subject: [PATCH 034/292] In the 'windirent' test module, use a macro for the hidden/system attribute checking. FossilOrigin-Name: a84a08d0716656dc0b26eafb1841c48d83c67ef2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_windirent.c | 11 ++++------- src/test_windirent.h | 11 +++++++++++ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 416ccf1043..9c7ce9a588 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssuperfluous\soption\sto\sTcl\s'lsort'\sin\sthe\svtabH\stest\sfile. -D 2017-01-18T22:19:01.508 +C In\sthe\s'windirent'\stest\smodule,\suse\sa\smacro\sfor\sthe\shidden/system\sattribute\schecking. +D 2017-01-18T22:47:42.937 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -446,8 +446,8 @@ F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c 5234b0c38bda5cb7dfc031db6a594af2cbcf7fb7 -F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01 +F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff +F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d3c91c1fb345fbcbfc60a897bebf771c795430c9 -R 2cfcfd69ff656c8eb0c9b18b58b2f78e +P b92cc6e58ae31cbe6600a522beb5485f7add04b2 +R 28b62fae1b56b071f03984b3c0c7d517 U mistachkin -Z e24ce1e7a409b1f16613abdf400a1354 +Z 8396ae361cfa963c7c2449732ae79bb0 diff --git a/manifest.uuid b/manifest.uuid index 936488379e..0a876335c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b92cc6e58ae31cbe6600a522beb5485f7add04b2 \ No newline at end of file +a84a08d0716656dc0b26eafb1841c48d83c67ef2 \ No newline at end of file diff --git a/src/test_windirent.c b/src/test_windirent.c index 450e4c3cc7..ca78d345d9 100644 --- a/src/test_windirent.c +++ b/src/test_windirent.c @@ -73,7 +73,7 @@ LPDIR opendir( } /* TODO: Remove this block to allow hidden and/or system files. */ - if( data.attrib&_A_HIDDEN || data.attrib&_A_SYSTEM ){ + if( is_filtered(data) ){ next: memset(&data, 0, sizeof(struct _finddata_t)); @@ -83,8 +83,7 @@ next: } /* TODO: Remove this block to allow hidden and/or system files. */ - if( data.attrib&_A_HIDDEN ) goto next; - if( data.attrib&_A_SYSTEM ) goto next; + if( is_filtered(data) ) goto next; } dirp->d_first.d_attributes = data.attrib; @@ -117,8 +116,7 @@ next: if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; /* TODO: Remove this block to allow hidden and/or system files. */ - if( data.attrib&_A_HIDDEN ) goto next; - if( data.attrib&_A_SYSTEM ) goto next; + if( is_filtered(data) ) goto next; dirp->d_next.d_ino++; dirp->d_next.d_attributes = data.attrib; @@ -162,8 +160,7 @@ next: } /* TODO: Remove this block to allow hidden and/or system files. */ - if( data.attrib&_A_HIDDEN ) goto next; - if( data.attrib&_A_SYSTEM ) goto next; + if( is_filtered(data) ) goto next; entry->d_ino = (ino_t)-1; /* not available */ entry->d_attributes = data.attrib; diff --git a/src/test_windirent.h b/src/test_windirent.h index be454988e3..578e2a7c22 100644 --- a/src/test_windirent.h +++ b/src/test_windirent.h @@ -92,6 +92,17 @@ struct DIR { DIRENT d_next; /* DIRENT constructed based on "_findnext". */ }; +/* +** Provide a macro, for use by the implementation, to determine if a +** particular directory entry should be skipped over when searching for +** the next directory entry that should be returned by the readdir() or +** readdir_r() functions. +*/ + +#ifndef is_filtered +# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) +#endif + /* ** Provide the function prototype for the POSIX compatiable getenv() ** function. This function is not thread-safe. From 2f31d0204960ccfae468cef94a0447311b9bb143 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 19 Jan 2017 18:20:36 +0000 Subject: [PATCH 035/292] In the 'vtshim' extension, avoid accessing freed memory when handling errors from xCreate/xConnect. FossilOrigin-Name: ffd559afd32dcdce9c733ebccdee88fda9b689cf --- ext/misc/vtshim.c | 2 ++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c index 01348e8d3e..0709a26a7f 100644 --- a/ext/misc/vtshim.c +++ b/ext/misc/vtshim.c @@ -95,6 +95,7 @@ static int vtshimCreate( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; @@ -133,6 +134,7 @@ static int vtshimConnect( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; diff --git a/manifest b/manifest index 9c7ce9a588..eab928e083 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s'windirent'\stest\smodule,\suse\sa\smacro\sfor\sthe\shidden/system\sattribute\schecking. -D 2017-01-18T22:47:42.937 +C In\sthe\s'vtshim'\sextension,\savoid\saccessing\sfreed\smemory\swhen\shandling\serrors\sfrom\sxCreate/xConnect. +D 2017-01-19T18:20:36.317 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -227,7 +227,7 @@ F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 -F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e +F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b92cc6e58ae31cbe6600a522beb5485f7add04b2 -R 28b62fae1b56b071f03984b3c0c7d517 +P a84a08d0716656dc0b26eafb1841c48d83c67ef2 +R 37615874230b85a181e9596f7f8a21ff U mistachkin -Z 8396ae361cfa963c7c2449732ae79bb0 +Z 99974373edae111c658f7548f0073df4 diff --git a/manifest.uuid b/manifest.uuid index 0a876335c4..01c9585ffd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a84a08d0716656dc0b26eafb1841c48d83c67ef2 \ No newline at end of file +ffd559afd32dcdce9c733ebccdee88fda9b689cf \ No newline at end of file From 8674e49214c8d9a0e23cf46c15d6bd4b74b00c4d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 19 Jan 2017 21:20:11 +0000 Subject: [PATCH 036/292] If compiled with SQLITE_INLINE_MEMCPY, all memcpy() calls are replaced with in-line code. With that change, cachegrind shows which memcpy() calls are taking the most time. This is a performance-measurement hack only and is not for production use. FossilOrigin-Name: 9ed38521617136223a667988aed40e25797faf84 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/loadext.c | 1 - src/resolve.c | 2 -- src/sqliteInt.h | 12 ++++++++++++ src/table.c | 2 -- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index eab928e083..6c86b1eb77 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s'vtshim'\sextension,\savoid\saccessing\sfreed\smemory\swhen\shandling\serrors\sfrom\sxCreate/xConnect. -D 2017-01-19T18:20:36.317 +C If\scompiled\swith\sSQLITE_INLINE_MEMCPY,\sall\smemcpy()\scalls\sare\sreplaced\swith\nin-line\scode.\s\sWith\sthat\schange,\scachegrind\sshows\swhich\smemcpy()\scalls\nare\staking\sthe\smost\stime.\s\sThis\sis\sa\sperformance-measurement\shack\sonly\sand\nis\snot\sfor\sproduction\suse. +D 2017-01-19T21:20:11.204 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -354,7 +354,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d +F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -388,17 +388,17 @@ F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb +F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h ce3e07c720b0cebc8887ea86b3b128da0913c5d3 +F src/sqliteInt.h 525c061ae9aafc8d4720a018d82f0936d9eee5ab F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 -F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 +F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 418f5e5e0840425a7e5b33f3600dccd378a57549 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a84a08d0716656dc0b26eafb1841c48d83c67ef2 -R 37615874230b85a181e9596f7f8a21ff -U mistachkin -Z 99974373edae111c658f7548f0073df4 +P ffd559afd32dcdce9c733ebccdee88fda9b689cf +R fad09dad35bb606057a537179db15d76 +U drh +Z 52e9f95dbec08dacc595dc3f0b2e0450 diff --git a/manifest.uuid b/manifest.uuid index 01c9585ffd..17bbd5a1dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffd559afd32dcdce9c733ebccdee88fda9b689cf \ No newline at end of file +9ed38521617136223a667988aed40e25797faf84 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 6011fd2bf8..3296be60dd 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -18,7 +18,6 @@ #endif #include "sqlite3ext.h" #include "sqliteInt.h" -#include #ifndef SQLITE_OMIT_LOAD_EXTENSION /* diff --git a/src/resolve.c b/src/resolve.c index dac73e5fa9..7d89b6fec5 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -15,8 +15,6 @@ ** table and column. */ #include "sqliteInt.h" -#include -#include /* ** Walk the expression tree pExpr and increase the aggregate function diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d5ac957d22..6344cac5f1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -511,6 +511,18 @@ #include #include +/* +** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY. +** This allows better measurements of where memcpy() is used when running +** cachegrind. But this macro version of memcpy() is very slow so it +** should not be used in production. This is a performance measurement +** hack only. +*/ +#ifdef SQLITE_INLINE_MEMCPY +# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\ + int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);} +#endif + /* ** If compiling for a processor that lacks floating point support, ** substitute integer for floating-point diff --git a/src/table.c b/src/table.c index a50d83ce63..c79255f990 100644 --- a/src/table.c +++ b/src/table.c @@ -17,8 +17,6 @@ ** if they are not used. */ #include "sqliteInt.h" -#include -#include #ifndef SQLITE_OMIT_GET_TABLE From 4df65fc20f617dbd01028e5987c1b1dcb0ca70aa Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 20 Jan 2017 00:40:26 +0000 Subject: [PATCH 037/292] Minor performance optimizations to sqlite3_blob_open() and sqlite3_blob_reopen(). FossilOrigin-Name: 52a61967d920047ea0b4409b79793e05c0128964 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 30 +++++++++++++++++------------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 6c86b1eb77..c04cd81412 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\scompiled\swith\sSQLITE_INLINE_MEMCPY,\sall\smemcpy()\scalls\sare\sreplaced\swith\nin-line\scode.\s\sWith\sthat\schange,\scachegrind\sshows\swhich\smemcpy()\scalls\nare\staking\sthe\smost\stime.\s\sThis\sis\sa\sperformance-measurement\shack\sonly\sand\nis\snot\sfor\sproduction\suse. -D 2017-01-19T21:20:11.204 +C Minor\sperformance\soptimizations\sto\ssqlite3_blob_open()\sand\nsqlite3_blob_reopen(). +D 2017-01-20T00:40:26.827 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -462,7 +462,7 @@ F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147 -F src/vdbeblob.c f4f98ea672b242f807c08c92c7faaa79e5091b65 +F src/vdbeblob.c fe3694fcc3cf2cad395416f5289bd0e935c2659d F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffd559afd32dcdce9c733ebccdee88fda9b689cf -R fad09dad35bb606057a537179db15d76 +P 9ed38521617136223a667988aed40e25797faf84 +R b4a609734c03fe3500908ef05ac99d6b U drh -Z 52e9f95dbec08dacc595dc3f0b2e0450 +Z 622609d2384fcdd28fbe219ab3e3b191 diff --git a/manifest.uuid b/manifest.uuid index 17bbd5a1dd..14542d0a86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ed38521617136223a667988aed40e25797faf84 \ No newline at end of file +52a61967d920047ea0b4409b79793e05c0128964 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 520d16c985..b2902ba321 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -57,12 +57,17 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; - /* Set the value of the SQL statements only variable to integer iRow. - ** This is done directly instead of using sqlite3_bind_int64() to avoid - ** triggering asserts related to mutexes. + /* Set the value of register r[1] in the SQL statement to integer iRow. + ** This is done directly as a performance optimization */ - assert( v->aVar[0].flags&MEM_Int ); - v->aVar[0].u.i = iRow; + v->aMem[1].flags = MEM_Int; + v->aMem[1].u.i = iRow; + + /* If the statement has been run before (and is paused at the OP_ResultRow) + ** then back it up to the point where it does the OP_SeekRowid. This could + ** have been down with an extra OP_Goto, but simply setting the program + ** counter is faster. */ + if( v->pc>3 ) v->pc = 3; rc = sqlite3_step(p->pStmt); if( rc==SQLITE_ROW ){ @@ -257,12 +262,11 @@ int sqlite3_blob_open( static const VdbeOpList openBlob[] = { {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ - {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ - {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 4 */ - {OP_ResultRow, 1, 0, 0}, /* 5 */ - {OP_Goto, 0, 2, 0}, /* 6 */ - {OP_Halt, 0, 0, 0}, /* 7 */ + /* blobSeekToRow() will initialize r[1] to the desired rowid */ + {OP_NotExists, 0, 5, 1}, /* 2: Seek the cursor to rowid=r[1] */ + {OP_Column, 0, 0, 1}, /* 3 */ + {OP_ResultRow, 1, 0, 0}, /* 4 */ + {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -306,9 +310,9 @@ int sqlite3_blob_open( */ aOp[1].p4type = P4_INT32; aOp[1].p4.i = pTab->nCol+1; - aOp[4].p2 = pTab->nCol; + aOp[3].p2 = pTab->nCol; - pParse->nVar = 1; + pParse->nVar = 0; pParse->nMem = 1; pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); From 508286701a5d2a8d893155c6cba1bf9b44cea0ca Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 20 Jan 2017 16:09:12 +0000 Subject: [PATCH 038/292] Get the "--testset rtree" option working on speedtest1. Add the --rtree, --lookaside, and --clang options to the speed-check.sh script. FossilOrigin-Name: 87b640c8d07a76b2bc7e896e01965cc09e06f77b --- main.mk | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/speedtest1.c | 28 ++++++++++------------------ tool/speed-check.sh | 19 ++++++++++++++++--- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/main.mk b/main.mk index e07b964d67..3b6e7f8609 100644 --- a/main.mk +++ b/main.mk @@ -893,7 +893,7 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c $(TOP)/test/wordcount.c sqlite3.c speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o - $(TCC) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) + $(TCCX) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ diff --git a/manifest b/manifest index c04cd81412..b018d640fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\soptimizations\sto\ssqlite3_blob_open()\sand\nsqlite3_blob_reopen(). -D 2017-01-20T00:40:26.827 +C Get\sthe\s"--testset\srtree"\soption\sworking\son\sspeedtest1.\s\sAdd\sthe\s--rtree,\n--lookaside,\sand\s--clang\soptions\sto\sthe\sspeed-check.sh\sscript. +D 2017-01-20T16:09:12.221 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -314,7 +314,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2cf5f0362c5687fd8e912c3a327b49a2e8ba0f9b +F main.mk 8f620f3418d7252a3f428f5aacf674b5507807a8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1131,7 +1131,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 4e8ea6165046f02e1cfe0f4700256e91c981ec10 +F test/speedtest1.c 02fe15bb784c5276a083ffe9969cc51e0bce7644 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1507,7 +1507,7 @@ F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a -F tool/speed-check.sh e6ca0695b047af64201ebe0ef452e423f55d78b1 +F tool/speed-check.sh 65ac2f5b00771b9dcefb95bebae1aab76c537ea3 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ed38521617136223a667988aed40e25797faf84 -R b4a609734c03fe3500908ef05ac99d6b +P 52a61967d920047ea0b4409b79793e05c0128964 +R 4e64e669335e5bcf79eed63ea8c1840b U drh -Z 622609d2384fcdd28fbe219ab3e3b191 +Z 6f7214599f293ddc9e6d41b975d33fdf diff --git a/manifest.uuid b/manifest.uuid index 14542d0a86..55a46f0885 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52a61967d920047ea0b4409b79793e05c0128964 \ No newline at end of file +87b640c8d07a76b2bc7e896e01965cc09e06f77b \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 02d55d5974..f5b79915c8 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -61,9 +61,6 @@ static const char zHelp[] = #if SQLITE_VERSION_NUMBER<3005000 # define sqlite3_int64 sqlite_int64 #endif -#ifdef SQLITE_ENABLE_RBU -# include "sqlite3rbu.h" -#endif /* All global state is held in this structure */ static struct Global { @@ -1181,10 +1178,10 @@ void testset_rtree(int p1, int p2){ unsigned mxCoord; unsigned x0, x1, y0, y1, z0, z1; unsigned iStep; - int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 ); + int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*500 ); mxCoord = 15000; - n = g.szTest*100; + n = g.szTest*500; speedtest1_begin_test(100, "%d INSERTs into an r-tree", n); speedtest1_exec("BEGIN"); speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)"); @@ -1207,11 +1204,11 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); speedtest1_begin_test(101, "Copy from rtree to a regular table"); - speedtest1_exec(" TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); + speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1"); speedtest1_end_test(); - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2"); iStep = mxCoord/n; @@ -1224,7 +1221,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); if( g.bVerify ){ - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(111, "Verify result from 1-D intersect slice queries"); speedtest1_prepare("SELECT count(*) FROM t1 WHERE x0>=?1 AND x1<=?2"); iStep = mxCoord/n; @@ -1240,7 +1237,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); } - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2"); iStep = mxCoord/n; @@ -1253,7 +1250,7 @@ void testset_rtree(int p1, int p2){ speedtest1_end_test(); if( g.bVerify ){ - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(121, "Verify result from 1-D overlap slice queries"); speedtest1_prepare("SELECT count(*) FROM t1 WHERE y1>=?1 AND y0<=?2"); iStep = mxCoord/n; @@ -1270,7 +1267,7 @@ void testset_rtree(int p1, int p2){ } - n = g.szTest*20; + n = g.szTest*100; speedtest1_begin_test(125, "%d custom geometry callback queries", n); sqlite3_rtree_geometry_callback(g.db, "xslice", xsliceGeometryCallback, 0); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE id MATCH xslice(?1,?2)"); @@ -1286,7 +1283,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); - n = g.szTest*80; + n = g.szTest*400; speedtest1_begin_test(130, "%d three-dimensional intersect box queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x1>=?1 AND x0<=?2" " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2"); @@ -1299,7 +1296,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); - n = g.szTest*100; + n = g.szTest*500; speedtest1_begin_test(140, "%d rowid queries", n); speedtest1_prepare("SELECT * FROM rt1 WHERE id=?1"); for(i=1; i<=n; i++){ @@ -1458,11 +1455,6 @@ int main(int argc, char **argv){ noSync = 1; }else if( strcmp(z,"notnull")==0 ){ g.zNN = "NOT NULL"; -#ifdef SQLITE_ENABLE_RBU - }else if( strcmp(z,"rbu")==0 ){ - sqlite3ota_create_vfs("rbu", 0); - sqlite3_vfs_register(sqlite3_vfs_find("rbu"), 1); -#endif }else if( strcmp(z,"pagesize")==0 ){ if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); pageSize = integerValue(argv[++i]); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index d6e7eb6d25..c6f587e177 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -23,6 +23,7 @@ NAME=$1 shift #CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" +CC=gcc SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64" SIZE=5 LEAN_OPTS="-DSQLITE_THREADSAFE=0" @@ -80,11 +81,19 @@ while test "$1" != ""; do --lean) CC_OPTS="$CC_OPTS $LEAN_OPTS" ;; + --clang) + CC=clang + ;; --heap) CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" ;; + --lookaside) + shift; + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2" + shift; + ;; --repeat) CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE" shift; @@ -94,6 +103,10 @@ while test "$1" != ""; do shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1" ;; + --rtree) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree" + CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE" + ;; *) CC_OPTS="$CC_OPTS $1" ;; @@ -105,15 +118,15 @@ echo "NAME = $NAME" | tee summary-$NAME.txt echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o -gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c +$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c size sqlite3.o | tee -a summary-$NAME.txt if test $doExplain -eq 1; then - gcc -g -Os -Wall -I. $CC_OPTS \ + $CC -g -Os -Wall -I. $CC_OPTS \ -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread fi SRC=./speedtest1.c -gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread +$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread ls -l speedtest1 | tee -a summary-$NAME.txt if test $doCachegrind -eq 1; then valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ From befcd8ad84f843cf384d754a8fd203b1945acb59 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 20 Jan 2017 16:46:20 +0000 Subject: [PATCH 039/292] Add option "--stats" to test program kvtest. Specifying --stats causes kvtest to output information similar to the shell tool option of the same name. FossilOrigin-Name: 90291327fc127671d9847a4a2ce1ed47a408cfc6 --- manifest | 14 +++--- manifest.uuid | 2 +- test/kvtest.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b018d640fc..882b5a775a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\s"--testset\srtree"\soption\sworking\son\sspeedtest1.\s\sAdd\sthe\s--rtree,\n--lookaside,\sand\s--clang\soptions\sto\sthe\sspeed-check.sh\sscript. -D 2017-01-20T16:09:12.221 +C Add\soption\s"--stats"\sto\stest\sprogram\skvtest.\sSpecifying\s--stats\scauses\skvtest\nto\soutput\sinformation\ssimilar\sto\sthe\sshell\stool\soption\sof\sthe\ssame\sname. +D 2017-01-20T16:46:20.421 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 2c66ddefcd03c2caa337f6dd79e6c82368af83df +F test/kvtest.c da3fddb003221d3d9726afd959d9c6017516fd02 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52a61967d920047ea0b4409b79793e05c0128964 -R 4e64e669335e5bcf79eed63ea8c1840b -U drh -Z 6f7214599f293ddc9e6d41b975d33fdf +P 87b640c8d07a76b2bc7e896e01965cc09e06f77b +R b9affb78955ec88bf34da37dcc1dd9ef +U dan +Z 3096b993434f8157eb2ffff3d1e5a264 diff --git a/manifest.uuid b/manifest.uuid index 55a46f0885..f7dbcbe1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -87b640c8d07a76b2bc7e896e01965cc09e06f77b \ No newline at end of file +90291327fc127671d9847a4a2ce1ed47a408cfc6 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 877605aced..8458ef919c 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -87,6 +87,7 @@ static const char zHelp[] = " --max-id N Maximum blob key to use\n" " --random Read blobs in a random order\n" " --start N Start reading with this blob key\n" +" --stats Output operating stats before exiting\n" ; /* Reference resources used */ @@ -369,6 +370,122 @@ static sqlite3_int64 timeOfDay(void){ return t; } +#ifdef __linux__ +/* +** Attempt to display I/O stats on Linux using /proc/PID/io +*/ +static void displayLinuxIoStats(FILE *out){ + FILE *in; + char z[200]; + sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); + in = fopen(z, "rb"); + if( in==0 ) return; + while( fgets(z, sizeof(z), in)!=0 ){ + static const struct { + const char *zPattern; + const char *zDesc; + } aTrans[] = { + { "rchar: ", "Bytes received by read():" }, + { "wchar: ", "Bytes sent to write():" }, + { "syscr: ", "Read() system calls:" }, + { "syscw: ", "Write() system calls:" }, + { "read_bytes: ", "Bytes read from storage:" }, + { "write_bytes: ", "Bytes written to storage:" }, + { "cancelled_write_bytes: ", "Cancelled write bytes:" }, + }; + int i; + for(i=0; i Date: Fri, 20 Jan 2017 16:47:34 +0000 Subject: [PATCH 040/292] Fix a typo in the help message for kvtest. FossilOrigin-Name: 8971d98f25a4f5fb060db8ed6a4b06f083122a50 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/kvtest.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 882b5a775a..5ca437cb49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soption\s"--stats"\sto\stest\sprogram\skvtest.\sSpecifying\s--stats\scauses\skvtest\nto\soutput\sinformation\ssimilar\sto\sthe\sshell\stool\soption\sof\sthe\ssame\sname. -D 2017-01-20T16:46:20.421 +C Fix\sa\stypo\sin\sthe\shelp\smessage\sfor\skvtest. +D 2017-01-20T16:47:34.130 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c da3fddb003221d3d9726afd959d9c6017516fd02 +F test/kvtest.c 371a2a0cb05840ed4a31c65c1d12a554c2fe556b F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 87b640c8d07a76b2bc7e896e01965cc09e06f77b -R b9affb78955ec88bf34da37dcc1dd9ef +P 90291327fc127671d9847a4a2ce1ed47a408cfc6 +R a902108b38beec2205eaeba771709e26 U dan -Z 3096b993434f8157eb2ffff3d1e5a264 +Z 09bc78c7c0c7ebb44e280cf03b306ae8 diff --git a/manifest.uuid b/manifest.uuid index f7dbcbe1e3..4b97c1dc13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90291327fc127671d9847a4a2ce1ed47a408cfc6 \ No newline at end of file +8971d98f25a4f5fb060db8ed6a4b06f083122a50 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 8458ef919c..2e997287e8 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -61,20 +61,20 @@ ** ./kvtest run x1 --count 10000 --max-id 1000000 */ static const char zHelp[] = -"Usage: kvhelp COMMAND ARGS...\n" +"Usage: kvtest COMMAND ARGS...\n" "\n" -" kvhelp init DBFILE --count N --size M --pagesize X\n" +" kvtest init DBFILE --count N --size M --pagesize X\n" "\n" " Generate a new test database file named DBFILE containing N\n" " BLOBs each of size M bytes. The page size of the new database\n" " file will be X\n" "\n" -" kvhelp export DBFILE DIRECTORY\n" +" kvtest export DBFILE DIRECTORY\n" "\n" " Export all the blobs in the kv table of DBFILE into separate\n" " files in DIRECTORY.\n" "\n" -" kvhelp run DBFILE [options]\n" +" kvtest run DBFILE [options]\n" "\n" " Run a performance test. DBFILE can be either the name of a\n" " database or a directory containing sample files. Options:\n" From cd645530157e1e754cf0fa37c3815a2253d01124 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 20 Jan 2017 20:43:14 +0000 Subject: [PATCH 041/292] Minor performance optimization and size reduction to the accessPayload() routine in btree.c. FossilOrigin-Name: 264e5c10d7144910b3223b64546567fa20e4bc65 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 15 ++++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 5ca437cb49..9780aa2959 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\shelp\smessage\sfor\skvtest. -D 2017-01-20T16:47:34.130 +C Minor\sperformance\soptimization\sand\ssize\sreduction\sto\sthe\saccessPayload()\nroutine\sin\sbtree.c. +D 2017-01-20T20:43:14.971 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 69966fb2c574954cd3216f09407c9a02c52d3bd7 +F src/btree.c e6b8f39d7dd1ad65f1747956de36f2595172a23d F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90291327fc127671d9847a4a2ce1ed47a408cfc6 -R a902108b38beec2205eaeba771709e26 -U dan -Z 09bc78c7c0c7ebb44e280cf03b306ae8 +P 8971d98f25a4f5fb060db8ed6a4b06f083122a50 +R 6803846831f043d39b9c85f99ff10309 +U drh +Z 62cd4db9c45f8ac36491823fd1143808 diff --git a/manifest.uuid b/manifest.uuid index 4b97c1dc13..6d1f724a7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8971d98f25a4f5fb060db8ed6a4b06f083122a50 \ No newline at end of file +264e5c10d7144910b3223b64546567fa20e4bc65 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7b56867646..fb05655912 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4523,16 +4523,14 @@ static int accessPayload( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } - if( rc==SQLITE_OK ){ - memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); - pCur->curFlags |= BTCF_ValidOvfl; - } + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; } /* If the overflow page-list cache has been allocated and the @@ -4547,8 +4545,8 @@ static int accessPayload( offset = (offset%ovflSize); } - for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ - + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ /* If required, populate the overflow page-list cache. */ if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ assert( pCur->aOverflow[iIdx]==0 @@ -4637,6 +4635,9 @@ static int accessPayload( amt -= a; pBuf += a; } + if( amt==0 ) break; + if( rc ) break; + iIdx++; } } From 36cae856eef6769c83da403bd73247c12474142f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Jan 2017 14:11:28 +0000 Subject: [PATCH 042/292] Remove an unnecessary sqlite3_bind_int64() call from sqlite3_blob_open(). Also other minor refactoring of the sqlite3_blob implementation. FossilOrigin-Name: 9d197a532349f4b1caf66bbed70ca46df86cb86f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 22 ++++++++++------------ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 9780aa2959..f0f41d4642 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\soptimization\sand\ssize\sreduction\sto\sthe\saccessPayload()\nroutine\sin\sbtree.c. -D 2017-01-20T20:43:14.971 +C Remove\san\sunnecessary\ssqlite3_bind_int64()\scall\sfrom\ssqlite3_blob_open().\nAlso\sother\sminor\srefactoring\sof\sthe\ssqlite3_blob\simplementation. +D 2017-01-21T14:11:28.343 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -462,7 +462,7 @@ F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147 -F src/vdbeblob.c fe3694fcc3cf2cad395416f5289bd0e935c2659d +F src/vdbeblob.c 824f360105b8cd43c2ec8c4611fd3b162a3a3eed F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8971d98f25a4f5fb060db8ed6a4b06f083122a50 -R 6803846831f043d39b9c85f99ff10309 +P 264e5c10d7144910b3223b64546567fa20e4bc65 +R 8ac11845f3680b4a63ed584c4ec0c997 U drh -Z 62cd4db9c45f8ac36491823fd1143808 +Z e8ceb26091bad5772496cc1502eaca7e diff --git a/manifest.uuid b/manifest.uuid index 6d1f724a7b..ca6ba2e6ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -264e5c10d7144910b3223b64546567fa20e4bc65 \ No newline at end of file +9d197a532349f4b1caf66bbed70ca46df86cb86f \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index b2902ba321..70a68b4015 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -23,10 +23,10 @@ */ typedef struct Incrblob Incrblob; struct Incrblob { - int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ - int iCol; /* Table column this handle is open on */ + u16 iCol; /* Table column this handle is open on */ + u8 isPureKV; /* True if pTab is a pure key/value table */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ @@ -117,7 +117,7 @@ int sqlite3_blob_open( const char *zTable, /* The table containing the blob */ const char *zColumn, /* The column containing the blob */ sqlite_int64 iRow, /* The row containing the glob */ - int flags, /* True -> read/write access, false -> read-only */ + int wrFlag, /* True -> read/write access, false -> read-only */ sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ ){ int nAttempt = 0; @@ -139,7 +139,7 @@ int sqlite3_blob_open( return SQLITE_MISUSE_BKPT; } #endif - flags = !!flags; /* flags = (flags ? 1 : 0); */ + wrFlag = !!wrFlag; /* wrFlag = (wrFlag ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); @@ -199,9 +199,8 @@ int sqlite3_blob_open( /* If the value is being opened for writing, check that the ** column is not indexed, and that it is not part of a foreign key. - ** It is against the rules to open a column to which either of these - ** descriptions applies for writing. */ - if( flags ){ + */ + if( wrFlag ){ const char *zFault = 0; Index *pIdx; #ifndef SQLITE_OMIT_FOREIGN_KEY @@ -272,7 +271,7 @@ int sqlite3_blob_open( int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; - sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); @@ -289,7 +288,7 @@ int sqlite3_blob_open( #else aOp[0].p1 = iDb; aOp[0].p2 = pTab->tnum; - aOp[0].p3 = flags; + aOp[0].p3 = wrFlag; sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); } if( db->mallocFailed==0 ){ @@ -297,7 +296,7 @@ int sqlite3_blob_open( /* Remove either the OP_OpenWrite or OpenRead. Set the P2 ** parameter of the other to pTab->tnum. */ - if( flags ) aOp[1].opcode = OP_OpenWrite; + if( wrFlag ) aOp[1].opcode = OP_OpenWrite; aOp[1].p2 = pTab->tnum; aOp[1].p3 = iDb; @@ -319,14 +318,13 @@ int sqlite3_blob_open( } } - pBlob->flags = flags; + pBlob->isPureKV = (pTab->nCol==2 && pTab->iPKey==0); pBlob->iCol = iCol; pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } - sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); } while( (++nAttempt) Date: Sat, 21 Jan 2017 15:55:41 +0000 Subject: [PATCH 043/292] In the kvtest.c test utility, reuse the buffer into which blobs are read, rather than reallocating it for each row. This is a closer match to how other test programs work, and thus provides a better comparison. FossilOrigin-Name: 0d1ad13a296b22d6fe36879b56f99bd6af1acd3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/kvtest.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f0f41d4642..17270cb285 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\ssqlite3_bind_int64()\scall\sfrom\ssqlite3_blob_open().\nAlso\sother\sminor\srefactoring\sof\sthe\ssqlite3_blob\simplementation. -D 2017-01-21T14:11:28.343 +C In\sthe\skvtest.c\stest\sutility,\sreuse\sthe\sbuffer\sinto\swhich\sblobs\sare\sread,\nrather\sthan\sreallocating\sit\sfor\seach\srow.\s\sThis\sis\sa\scloser\smatch\sto\show\nother\stest\sprograms\swork,\sand\sthus\sprovides\sa\sbetter\scomparison. +D 2017-01-21T15:55:41.656 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 371a2a0cb05840ed4a31c65c1d12a554c2fe556b +F test/kvtest.c 9e428931f0733b5ba65b6b5abd95b27320e875a3 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 264e5c10d7144910b3223b64546567fa20e4bc65 -R 8ac11845f3680b4a63ed584c4ec0c997 +P 9d197a532349f4b1caf66bbed70ca46df86cb86f +R 2ba22a751baba0a74113cf71f6292807 U drh -Z e8ceb26091bad5772496cc1502eaca7e +Z 28f64e9f53e7fb024862aee0e897381f diff --git a/manifest.uuid b/manifest.uuid index ca6ba2e6ad..1b9ab055af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d197a532349f4b1caf66bbed70ca46df86cb86f \ No newline at end of file +0d1ad13a296b22d6fe36879b56f99bd6af1acd3a \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 2e997287e8..d895af030f 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -516,6 +516,7 @@ static int runMain(int argc, char **argv){ int nData = 0; /* Bytes of data */ sqlite3_int64 nTotal = 0; /* Total data read */ unsigned char *pData; /* Content of the blob */ + int nAlloc = 0; /* Space allocated for pData[] */ assert( strcmp(argv[1],"run")==0 ); @@ -621,14 +622,16 @@ static int runMain(int argc, char **argv){ } if( rc==SQLITE_OK ){ nData = sqlite3_blob_bytes(pBlob); - pData = sqlite3_malloc( nData+1 ); + if( nAlloc Date: Sat, 21 Jan 2017 16:27:56 +0000 Subject: [PATCH 044/292] Change sqlite3_blob_reopen() to call sqlite3VdbeExec() directly rather than going through sqlite3_step(). Performance enhancement. FossilOrigin-Name: 347df3c1fd7322e7aacaf1e9f8be81830947c482 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 14 +++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 17270cb285..334c7a3604 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\skvtest.c\stest\sutility,\sreuse\sthe\sbuffer\sinto\swhich\sblobs\sare\sread,\nrather\sthan\sreallocating\sit\sfor\seach\srow.\s\sThis\sis\sa\scloser\smatch\sto\show\nother\stest\sprograms\swork,\sand\sthus\sprovides\sa\sbetter\scomparison. -D 2017-01-21T15:55:41.656 +C Change\ssqlite3_blob_reopen()\sto\scall\ssqlite3VdbeExec()\sdirectly\srather\sthan\ngoing\sthrough\ssqlite3_step().\s\sPerformance\senhancement. +D 2017-01-21T16:27:56.517 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -462,7 +462,7 @@ F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147 -F src/vdbeblob.c 824f360105b8cd43c2ec8c4611fd3b162a3a3eed +F src/vdbeblob.c 2159f36d2c3e7ed24e3ebe99a9a4b462248c0665 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9d197a532349f4b1caf66bbed70ca46df86cb86f -R 2ba22a751baba0a74113cf71f6292807 +P 0d1ad13a296b22d6fe36879b56f99bd6af1acd3a +R 6bebf342d641742884692400aca75850 U drh -Z 28f64e9f53e7fb024862aee0e897381f +Z fb104ae24cf906c93b7fa008d273f235 diff --git a/manifest.uuid b/manifest.uuid index 1b9ab055af..56e58c0ea3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d1ad13a296b22d6fe36879b56f99bd6af1acd3a \ No newline at end of file +347df3c1fd7322e7aacaf1e9f8be81830947c482 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 70a68b4015..1efd4c9243 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -26,7 +26,6 @@ struct Incrblob { int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ u16 iCol; /* Table column this handle is open on */ - u8 isPureKV; /* True if pTab is a pure key/value table */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ @@ -56,6 +55,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ int rc; /* Error code */ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; + sqlite3 *db = v->db; /* Set the value of register r[1] in the SQL statement to integer iRow. ** This is done directly as a performance optimization @@ -67,9 +67,14 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ ** then back it up to the point where it does the OP_SeekRowid. This could ** have been down with an extra OP_Goto, but simply setting the program ** counter is faster. */ - if( v->pc>3 ) v->pc = 3; - - rc = sqlite3_step(p->pStmt); + if( v->pc>3 ){ + v->pc = 3; + db->nVdbeExec++; + rc = sqlite3VdbeExec((Vdbe*)p->pStmt); + db->nVdbeExec--; + }else{ + rc = sqlite3_step(p->pStmt); + } if( rc==SQLITE_ROW ){ VdbeCursor *pC = v->apCsr[0]; u32 type = pC->aType[p->iCol]; @@ -318,7 +323,6 @@ int sqlite3_blob_open( } } - pBlob->isPureKV = (pTab->nCol==2 && pTab->iPKey==0); pBlob->iCol = iCol; pBlob->db = db; sqlite3BtreeLeaveAll(db); From 451e76d5b594d48d3450057ac542c7635b74b021 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Jan 2017 16:54:19 +0000 Subject: [PATCH 045/292] B-tree optimization: When seeking on a rowid table that has already been positioned, check to see if the new row happens to be the next row on the same leaf page. That is a reasonably common case, and if it is true it avoids a full binary search. FossilOrigin-Name: 8e5cfb2039126da7689c4b1c88760f10e1234eaf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 334c7a3604..cbd159d22a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\ssqlite3_blob_reopen()\sto\scall\ssqlite3VdbeExec()\sdirectly\srather\sthan\ngoing\sthrough\ssqlite3_step().\s\sPerformance\senhancement. -D 2017-01-21T16:27:56.517 +C B-tree\soptimization:\s\sWhen\sseeking\son\sa\srowid\stable\sthat\shas\salready\sbeen\npositioned,\scheck\sto\ssee\sif\sthe\snew\srow\shappens\sto\sbe\sthe\snext\srow\son\sthe\nsame\sleaf\spage.\s\sThat\sis\sa\sreasonably\scommon\scase,\sand\sif\sit\sis\strue\sit\navoids\sa\sfull\sbinary\ssearch. +D 2017-01-21T16:54:19.338 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c e6b8f39d7dd1ad65f1747956de36f2595172a23d +F src/btree.c 972b7e1346eb2f4369cf8cc43117c728e2bd9cd4 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d1ad13a296b22d6fe36879b56f99bd6af1acd3a -R 6bebf342d641742884692400aca75850 +P 347df3c1fd7322e7aacaf1e9f8be81830947c482 +R 3888200d85fb0a6e771ec0a3c9f2e5a8 U drh -Z fb104ae24cf906c93b7fa008d273f235 +Z 4b0826cf6fb92296cc5aa9f45d953141 diff --git a/manifest.uuid b/manifest.uuid index 56e58c0ea3..ac357ef297 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -347df3c1fd7322e7aacaf1e9f8be81830947c482 \ No newline at end of file +8e5cfb2039126da7689c4b1c88760f10e1234eaf \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fb05655912..8b095b57b0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5091,9 +5091,21 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; return SQLITE_OK; } - if( (pCur->curFlags & BTCF_AtLast)!=0 && pCur->info.nKeyinfo.nKeycurFlags & BTCF_AtLast)!=0 ){ + *pRes = -1; + return SQLITE_OK; + } + if( pCur->aiIdx[pCur->iPage]+1apPage[pCur->iPage]->nCell ){ + pCur->aiIdx[pCur->iPage]++; + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + } } } From 7f11afaba17fdc24ef01775613e4052a4100bbcf Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Jan 2017 21:47:54 +0000 Subject: [PATCH 046/292] A better implementation of the moveto-neighbor optimization that checks for nearby rows on adjacent pages. FossilOrigin-Name: 2c4ecb85a475b9063aa8a3bb517ac181a7ded649 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index cbd159d22a..95352ee96a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C B-tree\soptimization:\s\sWhen\sseeking\son\sa\srowid\stable\sthat\shas\salready\sbeen\npositioned,\scheck\sto\ssee\sif\sthe\snew\srow\shappens\sto\sbe\sthe\snext\srow\son\sthe\nsame\sleaf\spage.\s\sThat\sis\sa\sreasonably\scommon\scase,\sand\sif\sit\sis\strue\sit\navoids\sa\sfull\sbinary\ssearch. -D 2017-01-21T16:54:19.338 +C A\sbetter\simplementation\sof\sthe\smoveto-neighbor\soptimization\sthat\schecks\sfor\nnearby\srows\son\sadjacent\spages. +D 2017-01-21T21:47:54.621 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 972b7e1346eb2f4369cf8cc43117c728e2bd9cd4 +F src/btree.c f1b36bcfb1f9532ce64fe534153f11b8e2595d8b F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 347df3c1fd7322e7aacaf1e9f8be81830947c482 -R 3888200d85fb0a6e771ec0a3c9f2e5a8 +P 8e5cfb2039126da7689c4b1c88760f10e1234eaf +R 9f1865839aca85688e5829c506350f09 U drh -Z 4b0826cf6fb92296cc5aa9f45d953141 +Z fbf63ba541a6f5ec3e59c684016e63a0 diff --git a/manifest.uuid b/manifest.uuid index ac357ef297..478c310b27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e5cfb2039126da7689c4b1c88760f10e1234eaf \ No newline at end of file +2c4ecb85a475b9063aa8a3bb517ac181a7ded649 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8b095b57b0..b6bc66ea85 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5096,14 +5096,19 @@ int sqlite3BtreeMovetoUnpacked( *pRes = -1; return SQLITE_OK; } - if( pCur->aiIdx[pCur->iPage]+1apPage[pCur->iPage]->nCell ){ - pCur->aiIdx[pCur->iPage]++; - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - getCellInfo(pCur); - if( pCur->info.nKey==intKey ){ - *pRes = 0; - return SQLITE_OK; + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this ase, only a little more slowely */ + if( pCur->info.nKey+1==intKey && !pCur->skipNext ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, pRes); + if( rc ) return rc; + if( *pRes==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } } } } From eacb4412b1e15d5f0890888ea51d2c2af65abbf8 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 22 Jan 2017 00:11:07 +0000 Subject: [PATCH 047/292] Fix an initialized variable in kvtest. FossilOrigin-Name: ed62c5a6562262709128099f757ae60807a930d7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/kvtest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 334c7a3604..09f825982a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\ssqlite3_blob_reopen()\sto\scall\ssqlite3VdbeExec()\sdirectly\srather\sthan\ngoing\sthrough\ssqlite3_step().\s\sPerformance\senhancement. -D 2017-01-21T16:27:56.517 +C Fix\san\sinitialized\svariable\sin\skvtest. +D 2017-01-22T00:11:07.052 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 9e428931f0733b5ba65b6b5abd95b27320e875a3 +F test/kvtest.c 9f0efc313b7bbef9022e9e3fe699d99f5bdc7efa F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d1ad13a296b22d6fe36879b56f99bd6af1acd3a -R 6bebf342d641742884692400aca75850 +P 347df3c1fd7322e7aacaf1e9f8be81830947c482 +R 47630561b257ab51d84219dbb1624512 U drh -Z fb104ae24cf906c93b7fa008d273f235 +Z d718277d52e4c47c008f4f640e1286d7 diff --git a/manifest.uuid b/manifest.uuid index 56e58c0ea3..b5d52cb040 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -347df3c1fd7322e7aacaf1e9f8be81830947c482 \ No newline at end of file +ed62c5a6562262709128099f757ae60807a930d7 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index d895af030f..6847d03a44 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -515,7 +515,7 @@ static int runMain(int argc, char **argv){ sqlite3_int64 tmElapsed; /* Elapsed time */ int nData = 0; /* Bytes of data */ sqlite3_int64 nTotal = 0; /* Total data read */ - unsigned char *pData; /* Content of the blob */ + unsigned char *pData = 0; /* Content of the blob */ int nAlloc = 0; /* Space allocated for pData[] */ From 35f30d339847c340b8f3c3019b722d1d52835cc2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 22 Jan 2017 02:04:05 +0000 Subject: [PATCH 048/292] Fixes to documentation comments in the public header file. FossilOrigin-Name: 772dcb08f400f20d4dbfb74df39de78da24ee5fd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 09f825982a..0ba3b67a5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sinitialized\svariable\sin\skvtest. -D 2017-01-22T00:11:07.052 +C Fixes\sto\sdocumentation\scomments\sin\sthe\spublic\sheader\sfile. +D 2017-01-22T02:04:05.790 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -392,7 +392,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 -F src/sqlite.h.in e71655293c9bde26939496f3aac9d1821d2c07a2 +F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h 525c061ae9aafc8d4720a018d82f0936d9eee5ab @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 347df3c1fd7322e7aacaf1e9f8be81830947c482 -R 47630561b257ab51d84219dbb1624512 -U drh -Z d718277d52e4c47c008f4f640e1286d7 +P ed62c5a6562262709128099f757ae60807a930d7 +R 46519c248d38b7b1891c031deb62dd78 +U mistachkin +Z c8163604b14219f9ecc118cfce9ce251 diff --git a/manifest.uuid b/manifest.uuid index b5d52cb040..8d815a1227 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed62c5a6562262709128099f757ae60807a930d7 \ No newline at end of file +772dcb08f400f20d4dbfb74df39de78da24ee5fd \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7d7e70f901..87d5d121cb 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -576,7 +576,7 @@ int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. The +** flag indicates that a file cannot be deleted when open. The ** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on ** read-only media and cannot be changed even by processes with ** elevated privileges. @@ -726,6 +726,9 @@ struct sqlite3_file { **
  • [SQLITE_IOCAP_ATOMIC64K] **
  • [SQLITE_IOCAP_SAFE_APPEND] **
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of From 38305ab55ffa3f02c81e4632a5e24d56b97105ab Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 22 Jan 2017 16:34:35 +0000 Subject: [PATCH 049/292] Fix an uninitialized variable in the command-line shell. FossilOrigin-Name: 06b8001ade62bc59e6ae20f761167a81d85a4272 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0ba3b67a5d..9083b4695c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sdocumentation\scomments\sin\sthe\spublic\sheader\sfile. -D 2017-01-22T02:04:05.790 +C Fix\san\suninitialized\svariable\sin\sthe\scommand-line\sshell. +D 2017-01-22T16:34:35.577 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -391,7 +391,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 -F src/shell.c 6095531aa900decdaa765e0f3993fba7153c92c1 +F src/shell.c 59de9acab4423a536277653f2a9dcdd1307989f3 F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed62c5a6562262709128099f757ae60807a930d7 -R 46519c248d38b7b1891c031deb62dd78 -U mistachkin -Z c8163604b14219f9ecc118cfce9ce251 +P 772dcb08f400f20d4dbfb74df39de78da24ee5fd +R 2e31ebcfccc9f64daf4bbf7d84d2d498 +U drh +Z 386761af3eaa734f6328de143132fe67 diff --git a/manifest.uuid b/manifest.uuid index 8d815a1227..7dd55bd0de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -772dcb08f400f20d4dbfb74df39de78da24ee5fd \ No newline at end of file +06b8001ade62bc59e6ae20f761167a81d85a4272 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index b0928ce792..db4362dd32 100644 --- a/src/shell.c +++ b/src/shell.c @@ -5297,7 +5297,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; - sqlite3_vfs *pVfs; + sqlite3_vfs *pVfs = 0; if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ From 1b9fd2651bf2a04885038a991404e8623899b658 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 23 Jan 2017 07:06:27 +0000 Subject: [PATCH 050/292] Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX flag to custom tokenizer implementations. FossilOrigin-Name: 03c601344ed56b316bcc5fd02f6648b0009ba61b --- ext/fts5/fts5_expr.c | 2 +- ext/fts5/test/fts5synonym.test | 2 +- ext/fts5/test/fts5tokenizer.test | 39 ++++++++++++++++++++++++++++++++ manifest | 18 +++++++-------- manifest.uuid | 2 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9dcd88a5bc..209748bbf5 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1601,7 +1601,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ - int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); int n; sqlite3Fts5Dequote(z); n = (int)strlen(z); diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 185dda3ff4..6243663123 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -152,7 +152,7 @@ foreach {tn expr res} { 1 {abc} {"abc"} 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} - 4 {3*} {"3"|"iii"|"three" *} + 4 {3*} {"3" *} } { do_execsql_test 4.1.$tn { SELECT fts5_expr($expr, 'tokenize=tclnum') diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 9316d3c234..f68aa19a10 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -262,5 +262,44 @@ do_execsql_test 8.3 { brown dog fox jump lazi over quick the } +#------------------------------------------------------------------------- +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer +# implementation. +# +reset_db +proc tcl_create {args} { return "tcl_tokenize" } +sqlite3_fts5_create_tokenizer db tcl tcl_create +set ::flags [list] +proc tcl_tokenize {tflags text} { + lappend ::flags $tflags + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 9.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('xyz'); +} {} +do_test 9.1.2 { set ::flags } {document document} + +set ::flags [list] +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc} +do_test 9.2.2 { set ::flags } {query} + +set ::flags [list] +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc} +do_test 9.3.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {} +do_test 9.4.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} +do_test 9.5.2 { set ::flags } {query} + + finish_test diff --git a/manifest b/manifest index 9083b4695c..722072798a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\suninitialized\svariable\sin\sthe\scommand-line\sshell. -D 2017-01-22T16:34:35.577 +C Fix\sa\sproblem\spreventing\sFTS5\sfrom\scorrectly\spassing\sthe\sFTS5_TOKENIZE_PREFIX\nflag\sto\scustom\stokenizer\simplementations. +D 2017-01-23T07:06:27.840 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -102,7 +102,7 @@ F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c dc2cee9f56b1818b85df59304b8104a5dfb8ab60 +F ext/fts5/fts5_expr.c 33f0e8063ac7360bcd71c0ff0dcbacdc05fffe50 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 @@ -184,11 +184,11 @@ F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 F ext/fts5/test/fts5simple.test 5da9b15ed534eb0be9f279d8a2bb2e24d30e4e38 F ext/fts5/test/fts5simple2.test 00839031878f52391562594fdab0503e424ee071 F ext/fts5/test/fts5simple3.test 25faa8cb8ad470c6f01f670bcc1317c19a89f091 -F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 +F ext/fts5/test/fts5synonym.test f964f9a98580dec06c995d4638ecb36eee60ae4a F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 -F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 +F ext/fts5/test/fts5tokenizer.test f64ab5fe48fa1317ff6315264bfe26e73cbdefde F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 772dcb08f400f20d4dbfb74df39de78da24ee5fd -R 2e31ebcfccc9f64daf4bbf7d84d2d498 -U drh -Z 386761af3eaa734f6328de143132fe67 +P 06b8001ade62bc59e6ae20f761167a81d85a4272 +R 21442fb50b7964f67b800e2f23be99a7 +U dan +Z 98433cafc41c67731096a39c749dcd19 diff --git a/manifest.uuid b/manifest.uuid index 7dd55bd0de..6e356c14ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06b8001ade62bc59e6ae20f761167a81d85a4272 \ No newline at end of file +03c601344ed56b316bcc5fd02f6648b0009ba61b \ No newline at end of file From 65c4c0b02d4c7146659185d43815d6b99fc4df7b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 23 Jan 2017 15:58:09 +0000 Subject: [PATCH 051/292] Add the missing SQLITE_API symbol to test_delete.c (it is not added automatically as this file is not part of the amalgamation). FossilOrigin-Name: 7a4f512ddf9e7e718389c80930d6268ab598459c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_delete.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 722072798a..09b8ee5078 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sFTS5\sfrom\scorrectly\spassing\sthe\sFTS5_TOKENIZE_PREFIX\nflag\sto\scustom\stokenizer\simplementations. -D 2017-01-23T07:06:27.840 +C Add\sthe\smissing\sSQLITE_API\ssymbol\sto\stest_delete.c\s(it\sis\snot\sadded\nautomatically\sas\sthis\sfile\sis\snot\spart\sof\sthe\samalgamation). +D 2017-01-23T15:58:09.428 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -416,7 +416,7 @@ F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 F src/test_blob.c 6a4c7920d1d9c6cc0f7aa50c89c4f80016aeda83 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 83179ea845479b5be9a651d014649e3f2722a1fe -F src/test_delete.c 8499d4d323f2ec8e28301deb3d6ddd8eef8b8139 +F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06b8001ade62bc59e6ae20f761167a81d85a4272 -R 21442fb50b7964f67b800e2f23be99a7 +P 03c601344ed56b316bcc5fd02f6648b0009ba61b +R 9417217ad349116acb540369984c5408 U dan -Z 98433cafc41c67731096a39c749dcd19 +Z b2f31778ddace8b8427e4711f64f3edb diff --git a/manifest.uuid b/manifest.uuid index 6e356c14ae..e9d2ae6e41 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03c601344ed56b316bcc5fd02f6648b0009ba61b \ No newline at end of file +7a4f512ddf9e7e718389c80930d6268ab598459c \ No newline at end of file diff --git a/src/test_delete.c b/src/test_delete.c index 731f11e257..ca61965b27 100644 --- a/src/test_delete.c +++ b/src/test_delete.c @@ -80,7 +80,7 @@ static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){ ** Delete the database file identified by the string argument passed to this ** function. The string must contain a filename, not an SQLite URI. */ -int sqlite3_delete_database( +SQLITE_API int sqlite3_delete_database( const char *zFile /* File to delete */ ){ char *zBuf; /* Buffer to sprintf() filenames to */ From bace3240820b16b15547f3433bf5805f1656c012 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 23 Jan 2017 18:40:15 +0000 Subject: [PATCH 052/292] Add the --mmap option to the kvtest utility program. FossilOrigin-Name: 4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/kvtest.c | 9 +++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e45f5ecce..d96fa45b3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimization:\sTry\sto\savoid\sunnecessary\sbtree\ssearching\swhen\srepositioning\s\na\scursor\sto\sthe\snext\srow. -D 2017-01-23T16:56:18.240 +C Add\sthe\s--mmap\soption\sto\sthe\skvtest\sutility\sprogram. +D 2017-01-23T18:40:15.799 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 9f0efc313b7bbef9022e9e3fe699d99f5bdc7efa +F test/kvtest.c 87e6e974eb9e1502e00e77bc2831f3c9174a6dfb F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,8 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a4f512ddf9e7e718389c80930d6268ab598459c 2c4ecb85a475b9063aa8a3bb517ac181a7ded649 -R 29a4885c53e914cde79778fd2e323ab3 -T +closed 2c4ecb85a475b9063aa8a3bb517ac181a7ded649 +P ee793d30c1dc1f78f49e6230d17750eceedbd8ed +R c50065d103fe6e8d8104ae7a368dc7b9 U drh -Z 5710c3bf174cb3e3f9b43957024c6c5f +Z a1392fb7660aa1d68b81ecf426bbd2cf diff --git a/manifest.uuid b/manifest.uuid index 2e7c43d757..28766bd75a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee793d30c1dc1f78f49e6230d17750eceedbd8ed \ No newline at end of file +4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 6847d03a44..f87e4940ef 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -513,6 +513,7 @@ static int runMain(int argc, char **argv){ sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */ sqlite3_int64 tmStart; /* Start time */ sqlite3_int64 tmElapsed; /* Elapsed time */ + int mmapSize = 0; /* --mmap N argument */ int nData = 0; /* Bytes of data */ sqlite3_int64 nTotal = 0; /* Total data read */ unsigned char *pData = 0; /* Content of the blob */ @@ -535,6 +536,12 @@ static int runMain(int argc, char **argv){ if( nCount<1 ) fatalError("the --count must be positive"); continue; } + if( strcmp(z, "-mmap")==0 ){ + if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); + mmapSize = atoi(argv[++i]); + if( nCount<0 ) fatalError("the --mmap must be non-negative"); + continue; + } if( strcmp(z, "-max-id")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); iMax = atoi(argv[++i]); @@ -581,6 +588,8 @@ static int runMain(int argc, char **argv){ if( rc ){ fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); } + zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize); + sqlite3_exec(db, zSql, 0, 0, 0); zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache); sqlite3_exec(db, zSql, 0, 0, 0); sqlite3_free(zSql); From d1b2566b07744604a96593ea174f82e174eaa7f4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 23 Jan 2017 19:11:38 +0000 Subject: [PATCH 053/292] Document the --mmap option in the --help screen for kvtest. Enhance kvtest so that numeric arguments can have suffixes like "K" or "M". Add kvtest to the unix makefiles. FossilOrigin-Name: 175bda87288c7ce15b163316159f53a60822ccad --- Makefile.in | 9 ++++-- main.mk | 9 ++++-- manifest | 16 +++++------ manifest.uuid | 2 +- test/kvtest.c | 76 +++++++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/Makefile.in b/Makefile.in index b961070c23..d5fa831e6b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1177,8 +1177,13 @@ LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c + $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) + +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ + +kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c + $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) diff --git a/main.mk b/main.mk index 3b6e7f8609..17e6468e10 100644 --- a/main.mk +++ b/main.mk @@ -478,6 +478,8 @@ SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 DBFUZZ_OPT = +KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -892,8 +894,11 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \ $(TOP)/test/wordcount.c sqlite3.c -speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o - $(TCCX) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) +speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c + $(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) + +kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c + $(TCCX) -I. $(KV+OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ diff --git a/manifest b/manifest index d96fa45b3e..f8cd52fc5b 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Add\sthe\s--mmap\soption\sto\sthe\skvtest\sutility\sprogram. -D 2017-01-23T18:40:15.799 -F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e +C Document\sthe\s--mmap\soption\sin\sthe\s--help\sscreen\sfor\skvtest.\s\sEnhance\skvtest\sso\nthat\snumeric\sarguments\scan\shave\ssuffixes\slike\s"K"\sor\s"M".\s\sAdd\skvtest\sto\sthe\nunix\smakefiles. +D 2017-01-23T19:11:38.113 +F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -314,7 +314,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 8f620f3418d7252a3f428f5aacf674b5507807a8 +F main.mk afc52937b4e5fe08678e8d5a4fe4487d44e3bc61 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 87e6e974eb9e1502e00e77bc2831f3c9174a6dfb +F test/kvtest.c d2d7c434023498237cf731df21ca0687bf103858 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee793d30c1dc1f78f49e6230d17750eceedbd8ed -R c50065d103fe6e8d8104ae7a368dc7b9 +P 4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 +R 9f530d22e2b0a16555eacd054431a10f U drh -Z a1392fb7660aa1d68b81ecf426bbd2cf +Z 4864426d37a32c6f04be82da7889e371 diff --git a/manifest.uuid b/manifest.uuid index 28766bd75a..82e31c7d74 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 \ No newline at end of file +175bda87288c7ce15b163316159f53a60822ccad \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index f87e4940ef..c82a3f899e 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -85,6 +85,7 @@ static const char zHelp[] = " --count N Read N blobs\n" " --desc Read blobs in descending order\n" " --max-id N Maximum blob key to use\n" +" --mmap N Mmap as much as N bytes of DBFILE\n" " --random Read blobs in a random order\n" " --start N Start reading with this blob key\n" " --stats Output operating stats before exiting\n" @@ -132,6 +133,64 @@ static void fatalError(const char *zFormat, ...){ exit(1); } +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static int integerValue(const char *zArg){ + int v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( zArg[0]>='0' && zArg[0]<='9' ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; i65536 || ((pgsz-1)&pgsz)!=0 ){ fatalError("the --pagesize must be power of 2 between 512 and 65536"); } @@ -491,6 +550,7 @@ static int display_stats( #define ORDER_DESC 2 #define ORDER_RANDOM 3 + /* ** Run a performance test */ @@ -532,31 +592,31 @@ static int runMain(int argc, char **argv){ if( z[1]=='-' ) z++; if( strcmp(z, "-count")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); - nCount = atoi(argv[++i]); + nCount = integerValue(argv[++i]); if( nCount<1 ) fatalError("the --count must be positive"); continue; } if( strcmp(z, "-mmap")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); - mmapSize = atoi(argv[++i]); + mmapSize = integerValue(argv[++i]); if( nCount<0 ) fatalError("the --mmap must be non-negative"); continue; } if( strcmp(z, "-max-id")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); - iMax = atoi(argv[++i]); + iMax = integerValue(argv[++i]); if( iMax<1 ) fatalError("the --max-id must be positive"); continue; } if( strcmp(z, "-start")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); - iKey = atoi(argv[++i]); + iKey = integerValue(argv[++i]); if( iKey<1 ) fatalError("the --start must be positive"); continue; } if( strcmp(z, "-cache-size")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); - iCache = atoi(argv[++i]); + iCache = integerValue(argv[++i]); continue; } if( strcmp(z, "-random")==0 ){ From 210b0d0eb311e2fc2ece5c71a196811479f5f349 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Jan 2017 04:41:34 +0000 Subject: [PATCH 054/292] Ensure that sqlite3_blob_reopen() correctly handles short rows. Proposed fix for ticket [e6e962d6b0f06f46e]. Further testing needed. FossilOrigin-Name: 57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 4 +++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f8cd52fc5b..2325ce2625 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\sthe\s--mmap\soption\sin\sthe\s--help\sscreen\sfor\skvtest.\s\sEnhance\skvtest\sso\nthat\snumeric\sarguments\scan\shave\ssuffixes\slike\s"K"\sor\s"M".\s\sAdd\skvtest\sto\sthe\nunix\smakefiles. -D 2017-01-23T19:11:38.113 +C Ensure\sthat\ssqlite3_blob_reopen()\scorrectly\shandles\sshort\srows.\nProposed\sfix\sfor\sticket\s[e6e962d6b0f06f46e].\s\sFurther\stesting\sneeded. +D 2017-01-25T04:41:34.251 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -462,7 +462,7 @@ F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147 -F src/vdbeblob.c 2159f36d2c3e7ed24e3ebe99a9a4b462248c0665 +F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 @@ -1547,7 +1547,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 -R 9f530d22e2b0a16555eacd054431a10f +P 175bda87288c7ce15b163316159f53a60822ccad +R 344bedd0898a54c0d1255ff2528051f8 +T *branch * blob_reopen-fix +T *sym-blob_reopen-fix * +T -sym-trunk * U drh -Z 4864426d37a32c6f04be82da7889e371 +Z ac35e85a79d80d6e1f6100309e724651 diff --git a/manifest.uuid b/manifest.uuid index 82e31c7d74..1d4e697bcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -175bda87288c7ce15b163316159f53a60822ccad \ No newline at end of file +57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 1efd4c9243..810f78860f 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -77,7 +77,9 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ } if( rc==SQLITE_ROW ){ VdbeCursor *pC = v->apCsr[0]; - u32 type = pC->aType[p->iCol]; + u32 type = pC->nHdrParsed>p->iCol ? pC->aType[p->iCol] : 0; + testcase( pC->nHdrParsed==p->iCol ); + testcase( pC->nHdrParsed==p->iCol+1 ); if( type<12 ){ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", type==0?"null": type==7?"real": "integer" From 666d34c791536557855c3efd58a9a1c3ec7e59a0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Jan 2017 13:54:27 +0000 Subject: [PATCH 055/292] Fix SQLITEINT_H macro usage in two extensions. FossilOrigin-Name: 0803390c152141c9ab4e7a28406b2a5d72a5c2fa --- ext/misc/json1.c | 2 +- ext/userauth/userauth.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 327ad57500..3063f9f261 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -22,7 +22,7 @@ ** how JSONB might improve on that.) */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(_SQLITEINT_H_) +#if !defined(SQLITEINT_H) #include "sqlite3ext.h" #endif SQLITE_EXTENSION_INIT1 diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 6ce99053d3..a3d346c05c 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -22,7 +22,7 @@ ** directory as this file for additional information. */ #ifdef SQLITE_USER_AUTHENTICATION -#ifndef _SQLITEINT_H_ +#ifndef SQLITEINT_H # include "sqliteInt.h" #endif diff --git a/manifest b/manifest index f8cd52fc5b..01c84dc4e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\sthe\s--mmap\soption\sin\sthe\s--help\sscreen\sfor\skvtest.\s\sEnhance\skvtest\sso\nthat\snumeric\sarguments\scan\shave\ssuffixes\slike\s"K"\sor\s"M".\s\sAdd\skvtest\sto\sthe\nunix\smakefiles. -D 2017-01-23T19:11:38.113 +C Fix\sSQLITEINT_H\smacro\susage\sin\stwo\sextensions. +D 2017-01-25T13:54:27.195 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -213,7 +213,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c e0797ef1bfa637429ceab9140ecba620809ee02a +F ext/misc/json1.c 552a7d730863419e05dc947265b28bd411680e2e F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -310,7 +310,7 @@ F ext/session/sqlite3session.h 9345166bd8f80562145586cf817f707de5ecada2 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e +F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4948f7e6d2a1cfce36a7aab2f5b65be07c285ac3 -R 9f530d22e2b0a16555eacd054431a10f +P 175bda87288c7ce15b163316159f53a60822ccad +R a9fb164433cc898aa04fd742715e395e U drh -Z 4864426d37a32c6f04be82da7889e371 +Z 35bb9e1641d0b89bf56f9577622d86ef diff --git a/manifest.uuid b/manifest.uuid index 82e31c7d74..94c54c55e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -175bda87288c7ce15b163316159f53a60822ccad \ No newline at end of file +0803390c152141c9ab4e7a28406b2a5d72a5c2fa \ No newline at end of file From 585ce1923c7c45bd0bf4da4ad957b6421785d1f0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Jan 2017 14:58:27 +0000 Subject: [PATCH 056/292] Experimental enhancement to automatically trim NULL values from the end of records, for a reduced disk footprint. This change also involves increasing the P5 operand from 8 to 16 bits. FossilOrigin-Name: 118ded403b95050b74ae2b03919c43d614094a32 --- manifest | 24 +++++++++++++----------- manifest.uuid | 2 +- src/insert.c | 18 ++++++++++++++++++ src/sqliteInt.h | 1 + src/vdbe.c | 12 ++++++++++++ src/vdbe.h | 5 ++--- src/vdbeaux.c | 2 +- 7 files changed, 48 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index edea9fbb44..86115ea620 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\ssqlite3_blob_reopen()\scorrectly\shandles\sshort\srows.\s\nFix\sfor\sticket\s[e6e962d6b0f06f46e]. -D 2017-01-25T14:38:19.847 +C Experimental\senhancement\sto\sautomatically\strim\sNULL\svalues\sfrom\sthe\send\sof\nrecords,\sfor\sa\sreduced\sdisk\sfootprint.\s\sThis\schange\salso\sinvolves\sincreasing\nthe\sP5\soperand\sfrom\s8\sto\s16\sbits. +D 2017-01-25T14:58:27.321 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -352,7 +352,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 05e47e2de7b712a3a4148cd469e5f60873f5ef13 +F src/insert.c 07ccec2c1fb32dd2c5fbfd63426e98dc2a79a0fa F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -395,7 +395,7 @@ F src/shell.c 59de9acab4423a536277653f2a9dcdd1307989f3 F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 525c061ae9aafc8d4720a018d82f0936d9eee5ab +F src/sqliteInt.h dc587b0727c2bc888d78d200d891620080567691 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -457,11 +457,11 @@ F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c c7add5978cb84ae3a7bcb16f8b56cb3bbdf04b7e -F src/vdbe.h b0866e4191f096f1c987a84b042c3599bdf5423b +F src/vdbe.c b1f2448184fa58c66fc94591004b1258964ef9da +F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 -F src/vdbeaux.c 35c9a9908174e5a26c96d15e1f98214814a39147 +F src/vdbeaux.c 8a2446741a2ec1072e744bc50f69b8b9e6c36592 F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1547,8 +1547,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0803390c152141c9ab4e7a28406b2a5d72a5c2fa 57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2 -R d15a5cee6a51c7118a4d78b7d0cbd2de -T +closed 57d8dad35c2a9ab635e954dce7f3986ae1ca8ed2 +P 8cd1a4451cce1fe28f462800e2be1dee1735c0d0 +R 8038c1c21360d07be65589e30167ecdf +T *branch * trim-nulls +T *sym-trim-nulls * +T -sym-trunk * U drh -Z f9af43b146da8d8246d0b97014ae0d3a +Z d170ee380cb17c0b674fe48dc7bc03d5 diff --git a/manifest.uuid b/manifest.uuid index d15e64c7de..6aea75dca3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cd1a4451cce1fe28f462800e2be1dee1735c0d0 \ No newline at end of file +118ded403b95050b74ae2b03919c43d614094a32 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 93c22ae3f5..5bb8ba2ff9 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1668,6 +1668,23 @@ void sqlite3GenerateConstraintChecks( VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol; i>1 && pTab->aCol[i-1].pDflt==0; i--){} + sqlite3VdbeChangeP5(v, i); +} + /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. @@ -1727,6 +1744,7 @@ void sqlite3CompleteInsertion( regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); if( !bAffinityDone ){ sqlite3TableAffinity(v, pTab, 0); sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6344cac5f1..ceae4dcb1f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3790,6 +3790,7 @@ int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3ResolvePartIdxLabel(Parse*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*,int*); +void sqlite3SetMakeRecordP5(Vdbe*,Table*); void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); void sqlite3BeginWriteOperation(Parse*, int, int); diff --git a/src/vdbe.c b/src/vdbe.c index cbb7867512..88010a9a6f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2777,6 +2777,18 @@ case OP_MakeRecord: { }while( zAffinity[0] ); } + /* NULLs can be safely trimmed from the end of the record, as long as + ** as the schema format is 2 or more and none of the omitted columns + ** have a non-NULL default value. Also, the record must be left with + ** at least one field. If P5>0 then it will be one more than the + ** index of the right-most column with a non-NULL default value */ + if( pOp->p5 ){ + while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){ + pLast--; + nField--; + } + } + /* Loop through the elements that will make up the record to figure ** out how much space is required for the new record. */ diff --git a/src/vdbe.h b/src/vdbe.h index feaf116807..a35f3be344 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -41,8 +41,7 @@ typedef struct SubProgram SubProgram; struct VdbeOp { u8 opcode; /* What operation to perform */ signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 notUsed1; - u8 p5; /* Fifth parameter is an unsigned character */ + u16 p5; /* Fifth parameter is an unsigned 16-bit integer */ int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ @@ -194,7 +193,7 @@ void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); -void sqlite3VdbeChangeP5(Vdbe*, u8 P5); +void sqlite3VdbeChangeP5(Vdbe*, u16 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); int sqlite3VdbeChangeToNoop(Vdbe*, int addr); int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 85d273f1ec..cd423ed5b5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -783,7 +783,7 @@ void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ sqlite3VdbeGetOp(p,addr)->p3 = val; } -void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ +void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ assert( p->nOp>0 || p->db->mallocFailed ); if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } From 2a86c1962c04e6f3fbec04121dc7b69e2a702648 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Jan 2017 17:44:13 +0000 Subject: [PATCH 057/292] Fix a problem with the pre-update hook on this branch. FossilOrigin-Name: 6fc4fbfa29cfa795edf32e4a1f2d0eceb3007f68 --- manifest | 23 ++++++++++------------- manifest.uuid | 2 +- src/vdbeapi.c | 10 ++++------ src/vdbeaux.c | 8 ++++---- test/hook.test | 36 ++++++++++++++++++++++++++++++++++++ test/tester.tcl | 2 +- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 86115ea620..0fa8005bed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\senhancement\sto\sautomatically\strim\sNULL\svalues\sfrom\sthe\send\sof\nrecords,\sfor\sa\sreduced\sdisk\sfootprint.\s\sThis\schange\salso\sinvolves\sincreasing\nthe\sP5\soperand\sfrom\s8\sto\s16\sbits. -D 2017-01-25T14:58:27.321 +C Fix\sa\sproblem\swith\sthe\spre-update\shook\son\sthis\sbranch. +D 2017-01-25T17:44:13.969 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -460,8 +460,8 @@ F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 F src/vdbe.c b1f2448184fa58c66fc94591004b1258964ef9da F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e -F src/vdbeapi.c d6ebaa465f070eb1af8ba4e7b34583ece87bdd24 -F src/vdbeaux.c 8a2446741a2ec1072e744bc50f69b8b9e6c36592 +F src/vdbeapi.c 9a44ed2f4fcb5a10dec9da5af95293d31830f268 +F src/vdbeaux.c 7c19b78999faae833e1be66dfa4e3deaf334d739 F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -832,7 +832,7 @@ F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 3a01b876691f9151d3e44562354f7d663ff90fce +F test/hook.test 09b8ce2226776b10bf74344e67d81291a49801f6 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -1163,7 +1163,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 4ce5afd5e192db4cae178e1a983b060e0f08c5d6 +F test/tester.tcl 2a49c1aff731f380ea640106377e19611a1443ae F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1547,10 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8cd1a4451cce1fe28f462800e2be1dee1735c0d0 -R 8038c1c21360d07be65589e30167ecdf -T *branch * trim-nulls -T *sym-trim-nulls * -T -sym-trunk * -U drh -Z d170ee380cb17c0b674fe48dc7bc03d5 +P 118ded403b95050b74ae2b03919c43d614094a32 +R b2da1f7f2be6fc581980d1f7d80d3a07 +U dan +Z e5faacbef3e7f4e557b616df3270ec25 diff --git a/manifest.uuid b/manifest.uuid index 6aea75dca3..eaecf9f4a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -118ded403b95050b74ae2b03919c43d614094a32 \ No newline at end of file +6fc4fbfa29cfa795edf32e4a1f2d0eceb3007f68 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 32794de9a0..91edd34022 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1776,13 +1776,11 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } p->pNewUnpacked = pUnpack; } - if( iIdx>=pUnpack->nField ){ + pMem = &pUnpack->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else if( iIdx>=pUnpack->nField ){ pMem = (sqlite3_value *)columnNullValue(); - }else{ - pMem = &pUnpack->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey2); - } } }else{ /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required diff --git a/src/vdbeaux.c b/src/vdbeaux.c index cd423ed5b5..44f2ad38f4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -4585,10 +4585,10 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** This function is used to free UnpackedRecord structures allocated by ** the vdbeUnpackRecord() function found in vdbeapi.c. */ -static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){ +static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ if( p ){ int i; - for(i=0; inField; i++){ + for(i=0; iaMem[i]; if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); } @@ -4647,8 +4647,8 @@ void sqlite3VdbePreUpdateHook( db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pNewUnpacked); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ diff --git a/test/hook.test b/test/hook.test index 8f095c9e8d..c576449502 100644 --- a/test/hook.test +++ b/test/hook.test @@ -854,4 +854,40 @@ do_preupdate_test 8.1 { } { } +#------------------------------------------------------------------------- +reset_db +db preupdate hook preupdate_hook +do_execsql_test 9.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(a, b INTEGER PRIMARY KEY); +} +do_preupdate_test 9.1 { + INSERT INTO t1 VALUES(456, NULL, NULL); +} { + INSERT main t1 456 456 0 456 {} {} +} +do_execsql_test 9.2 { + ALTER TABLE t1 ADD COLUMN d; +} +do_preupdate_test 9.3 { + INSERT INTO t1(a, b, c) VALUES(457, NULL, NULL); +} { + INSERT main t1 457 457 0 457 {} {} {} +} +do_preupdate_test 9.4 { + DELETE FROM t1 WHERE a=456 +} { + DELETE main t1 456 456 0 456 {} {} {} +} +do_preupdate_test 9.5 { + INSERT INTO t2 DEFAULT VALUES; +} { + INSERT main t2 1 1 0 {} 1 +} +do_preupdate_test 9.6 { + INSERT INTO t1 DEFAULT VALUES; +} { + INSERT main t1 458 458 0 458 {} {} {} +} + finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 814788ba45..8cc501a182 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -461,7 +461,7 @@ if {[info exists cmdlinearg]==0} { } {^-+backtrace=.+$} { foreach {dummy cmdlinearg(backtrace)} [split $a =] break - sqlite3_memdebug_backtrace $value + sqlite3_memdebug_backtrace $cmdlinearg(backtrace) } {^-+binarylog=.+$} { foreach {dummy cmdlinearg(binarylog)} [split $a =] break From d47e1ccb57ee3693ddf91162a0f38934007fef7e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Jan 2017 18:12:46 +0000 Subject: [PATCH 058/292] Fix a test script problem in exclusive2.test causing it to fail on this branch. FossilOrigin-Name: f66614dc78e32d2d369518200b3322cd97990ffe --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/exclusive2.test | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 0fa8005bed..88faa9a140 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\spre-update\shook\son\sthis\sbranch. -D 2017-01-25T17:44:13.969 +C Fix\sa\stest\sscript\sproblem\sin\sexclusive2.test\scausing\sit\sto\sfail\son\sthis\nbranch. +D 2017-01-25T18:12:46.110 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -679,7 +679,7 @@ F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 -F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 +F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 118ded403b95050b74ae2b03919c43d614094a32 -R b2da1f7f2be6fc581980d1f7d80d3a07 +P 6fc4fbfa29cfa795edf32e4a1f2d0eceb3007f68 +R c06506894b1f7b9de189bb73f3ac7abf U dan -Z e5faacbef3e7f4e557b616df3270ec25 +Z 47f4a5e852dc6941a6a828eee1ba5cee diff --git a/manifest.uuid b/manifest.uuid index eaecf9f4a3..947e5dcd6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6fc4fbfa29cfa795edf32e4a1f2d0eceb3007f68 \ No newline at end of file +f66614dc78e32d2d369518200b3322cd97990ffe \ No newline at end of file diff --git a/test/exclusive2.test b/test/exclusive2.test index 712363e762..92fcd76ab3 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -121,13 +121,13 @@ do_test exclusive2-1.1 { execsql { BEGIN; CREATE TABLE t1(a, b); - INSERT INTO t1(a) VALUES(randstr(10, 400)); - INSERT INTO t1(a) VALUES(randstr(10, 400)); - INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1; - INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1; - INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1; - INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1; - INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1; + INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0); + INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0); + INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1; + INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1; + INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1; + INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1; + INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1; COMMIT; SELECT count(*) FROM t1; } @@ -154,7 +154,7 @@ do_test exclusive2-1.4 { } $::sig do_test exclusive2-1.5 { execsql { - UPDATE t1 SET b=a, a=NULL; + UPDATE t1 SET b=a, a=0; } db2 expr {[t1sig db2] eq $::sig} } 0 From 7271d7a19c2d651146fed05680e1b393c4901d2c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Jan 2017 18:53:27 +0000 Subject: [PATCH 059/292] Fix another pre-update hook issue, this time in sqlite3preupdate_old(). FossilOrigin-Name: c7651d21bfdfd9b8cf04b26e0264bc58c03d247f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 18 ++++++++---------- test/hook.test | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 88faa9a140..e160f9e3f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\sin\sexclusive2.test\scausing\sit\sto\sfail\son\sthis\nbranch. -D 2017-01-25T18:12:46.110 +C Fix\sanother\spre-update\shook\sissue,\sthis\stime\sin\ssqlite3preupdate_old(). +D 2017-01-25T18:53:27.729 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -460,7 +460,7 @@ F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 F src/vdbe.c b1f2448184fa58c66fc94591004b1258964ef9da F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e -F src/vdbeapi.c 9a44ed2f4fcb5a10dec9da5af95293d31830f268 +F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 F src/vdbeaux.c 7c19b78999faae833e1be66dfa4e3deaf334d739 F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd @@ -832,7 +832,7 @@ F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 09b8ce2226776b10bf74344e67d81291a49801f6 +F test/hook.test f6a48d33817f0ca1a39a4d6605fe7e9da8077522 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6fc4fbfa29cfa795edf32e4a1f2d0eceb3007f68 -R c06506894b1f7b9de189bb73f3ac7abf +P f66614dc78e32d2d369518200b3322cd97990ffe +R b9395615953abce115528c9bb805cca4 U dan -Z 47f4a5e852dc6941a6a828eee1ba5cee +Z c5b3bc3bd2550d6ddb6d6d5566bf3773 diff --git a/manifest.uuid b/manifest.uuid index 947e5dcd6a..5de3845661 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f66614dc78e32d2d369518200b3322cd97990ffe \ No newline at end of file +c7651d21bfdfd9b8cf04b26e0264bc58c03d247f \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 91edd34022..7ecdac87c3 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1660,6 +1660,7 @@ static UnpackedRecord *vdbeUnpackRecord( */ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ PreUpdate *p = db->pPreUpdate; + Mem *pMem; int rc = SQLITE_OK; /* Test that this call is being made from within an SQLITE_DELETE or @@ -1693,17 +1694,14 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ p->aRecord = aRec; } - if( iIdx>=p->pUnpacked->nField ){ + pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else if( iIdx>=p->pUnpacked->nField ){ *ppValue = (sqlite3_value *)columnNullValue(); - }else{ - Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; - *ppValue = &p->pUnpacked->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey1); - }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ - sqlite3VdbeMemRealify(pMem); - } + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & MEM_Int ){ + sqlite3VdbeMemRealify(pMem); } } diff --git a/test/hook.test b/test/hook.test index c576449502..0c24ec7313 100644 --- a/test/hook.test +++ b/test/hook.test @@ -890,4 +890,19 @@ do_preupdate_test 9.6 { INSERT main t1 458 458 0 458 {} {} {} } + +do_execsql_test 10.0 { + CREATE TABLE t3(a, b INTEGER PRIMARY KEY); +} +do_preupdate_test 10.1 { + INSERT INTO t3 DEFAULT VALUES +} { + INSERT main t3 1 1 0 {} 1 +} +do_execsql_test 10.2 { SELECT * FROM t3 } {{} 1} +do_preupdate_test 10.3 { + DELETE FROM t3 WHERE b=1 +} {DELETE main t3 1 1 0 {} 1} + + finish_test From 87f500ce43c2fa0b5797b92149513c530a3983f6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 00:58:27 +0000 Subject: [PATCH 060/292] Modify the ICU extension to use a static initializer, as VC++ complains about a dynamic initialization. Maybe the dynamic structure initialization is a GCC extension. FossilOrigin-Name: 50e60cb44fd3687dde5551d02bad60c323beaabc --- ext/icu/icu.c | 44 +++++++++++++++++++++----------------------- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/ext/icu/icu.c b/ext/icu/icu.c index d2beaa3353..50518d08a7 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -493,38 +493,36 @@ static void icuLoadCollation( ** Register the ICU extension functions with database db. */ int sqlite3IcuInit(sqlite3 *db){ - struct IcuScalar { + static const struct IcuScalar { const char *zName; /* Function name */ - int nArg; /* Number of arguments */ - int enc; /* Optimal text encoding */ - void *pContext; /* sqlite3_user_data() context */ + unsigned char nArg; /* Number of arguments */ + unsigned short enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, - - {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - - {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + {"icu_load_collation", 2, SQLITE_UTF8, 255, icuLoadCollation}, + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, }; - int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ - struct IcuScalar *p = &scalars[i]; + const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( - db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + db, p->zName, p->nArg, p->enc, + p->iContext==255 ? (void*)db : (void*)(((char*)0)+p->iContext), + p->xFunc, 0, 0 ); } diff --git a/manifest b/manifest index b6cd8613f4..70cd2eef66 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Trim\sNULL\svalues\soff\sthe\send\sof\srecords\swhen\sthe\sSQLITE_ENABLE_TRIM_NULLS\ncompile-time\soption\sis\sused.\s\sIncrease\sthe\ssize\sof\sthe\sP5\soperand\sto\s16\sbits.\nFix\sa\sproblem\swith\sshort\srecords\sin\sthe\ssessions\sextension. -D 2017-01-25T20:55:11.502 +C Modify\sthe\sICU\sextension\sto\suse\sa\sstatic\sinitializer,\sas\sVC++\scomplains\sabout\na\sdynamic\sinitialization.\s\sMaybe\sthe\sdynamic\sstructure\sinitialization\sis\sa\nGCC\sextension. +D 2017-01-26T00:58:27.622 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -202,7 +202,7 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 -F ext/icu/icu.c 03ff6f90f3004a7e5a86205b581b2b7035ebf6e1 +F ext/icu/icu.c bae1dde13091b425bbed119a09a11aee14644764 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d @@ -1547,8 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8cd1a4451cce1fe28f462800e2be1dee1735c0d0 c7651d21bfdfd9b8cf04b26e0264bc58c03d247f -R 97e00d1e041d69da0e0a6af8a75ee1a0 -T +closed c7651d21bfdfd9b8cf04b26e0264bc58c03d247f +P 4801bd59a01dcc11a3eb9e776e7599b36f162d2a +R 11380cdeca896e71dff2973b03351379 U drh -Z 857c1b54405ac0784389a22078eb728f +Z dca6dbb559e3b4e2e7d7b023b68a9123 diff --git a/manifest.uuid b/manifest.uuid index 854e8f8a47..6734156cad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4801bd59a01dcc11a3eb9e776e7599b36f162d2a \ No newline at end of file +50e60cb44fd3687dde5551d02bad60c323beaabc \ No newline at end of file From 2b15f6e11bad7e8faeef497aadf6b5df8bf229c4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 01:54:39 +0000 Subject: [PATCH 061/292] Enhancements to the kvtest utility program. Add the --jmode option. Improved output formatting. FossilOrigin-Name: 62a4851ccf88837d1c16dae8204f7f264e80e3c9 --- manifest | 12 +++++----- manifest.uuid | 2 +- test/kvtest.c | 61 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 70cd2eef66..46dc00f8c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sICU\sextension\sto\suse\sa\sstatic\sinitializer,\sas\sVC++\scomplains\sabout\na\sdynamic\sinitialization.\s\sMaybe\sthe\sdynamic\sstructure\sinitialization\sis\sa\nGCC\sextension. -D 2017-01-26T00:58:27.622 +C Enhancements\sto\sthe\skvtest\sutility\sprogram.\s\sAdd\sthe\s--jmode\soption.\s\nImproved\soutput\sformatting. +D 2017-01-26T01:54:39.591 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -898,7 +898,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c d2d7c434023498237cf731df21ca0687bf103858 +F test/kvtest.c dc6e5e9066fa5e19f7368c7dfbe8c6b9642a706f F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4801bd59a01dcc11a3eb9e776e7599b36f162d2a -R 11380cdeca896e71dff2973b03351379 +P 50e60cb44fd3687dde5551d02bad60c323beaabc +R 6c49e01bee6ced9ad603e50cc8404852 U drh -Z dca6dbb559e3b4e2e7d7b023b68a9123 +Z 5b3363ef063097392f2b7107d7fe03e8 diff --git a/manifest.uuid b/manifest.uuid index 6734156cad..7d6c267913 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50e60cb44fd3687dde5551d02bad60c323beaabc \ No newline at end of file +62a4851ccf88837d1c16dae8204f7f264e80e3c9 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index c82a3f899e..765a7ee968 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -79,16 +79,17 @@ static const char zHelp[] = " Run a performance test. DBFILE can be either the name of a\n" " database or a directory containing sample files. Options:\n" "\n" -" --asc Read blobs in ascending order\n" -" --blob-api Use the BLOB API\n" -" --cache-size N Database cache size\n" -" --count N Read N blobs\n" -" --desc Read blobs in descending order\n" -" --max-id N Maximum blob key to use\n" -" --mmap N Mmap as much as N bytes of DBFILE\n" -" --random Read blobs in a random order\n" -" --start N Start reading with this blob key\n" -" --stats Output operating stats before exiting\n" +" --asc Read blobs in ascending order\n" +" --blob-api Use the BLOB API\n" +" --cache-size N Database cache size\n" +" --count N Read N blobs\n" +" --desc Read blobs in descending order\n" +" --max-id N Maximum blob key to use\n" +" --mmap N Mmap as much as N bytes of DBFILE\n" +" --jmode MODE Set MODE journal mode prior to starting\n" +" --random Read blobs in a random order\n" +" --start N Start reading with this blob key\n" +" --stats Output operating stats before exiting\n" ; /* Reference resources used */ @@ -539,9 +540,6 @@ static int display_stats( displayLinuxIoStats(out); #endif - /* Do not remove this machine readable comment: extra-stats-output-here */ - - fprintf(out, "\n"); return 0; } @@ -562,7 +560,7 @@ static int runMain(int argc, char **argv){ int nCount = 1000; /* Number of blob fetch operations */ int nExtra = 0; /* Extra cycles */ int iKey = 1; /* Next blob key */ - int iMax = 1000; /* Largest allowed key */ + int iMax = 0; /* Largest allowed key */ int iPagesize = 0; /* Database page size */ int iCache = 1000; /* Database cache size in kibibytes */ int bBlobApi = 0; /* Use the incremental blob I/O API */ @@ -578,6 +576,7 @@ static int runMain(int argc, char **argv){ sqlite3_int64 nTotal = 0; /* Total data read */ unsigned char *pData = 0; /* Content of the blob */ int nAlloc = 0; /* Space allocated for pData[] */ + const char *zJMode = 0; /* Journal mode */ assert( strcmp(argv[1],"run")==0 ); @@ -605,7 +604,6 @@ static int runMain(int argc, char **argv){ if( strcmp(z, "-max-id")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); iMax = integerValue(argv[++i]); - if( iMax<1 ) fatalError("the --max-id must be positive"); continue; } if( strcmp(z, "-start")==0 ){ @@ -619,6 +617,11 @@ static int runMain(int argc, char **argv){ iCache = integerValue(argv[++i]); continue; } + if( strcmp(z, "-jmode")==0 ){ + if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); + zJMode = argv[++i]; + continue; + } if( strcmp(z, "-random")==0 ){ eOrder = ORDER_RANDOM; continue; @@ -667,8 +670,29 @@ static int runMain(int argc, char **argv){ } sqlite3_finalize(pStmt); pStmt = 0; + if( zJMode ){ + zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode); + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + }else{ + zJMode = "???"; + } + sqlite3_finalize(pStmt); + if( iMax<=0 ){ + sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iMax = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + } + pStmt = 0; sqlite3_exec(db, "BEGIN", 0, 0, 0); } + if( iMax<=0 ) iMax = 1000; for(i=0; i Date: Thu, 26 Jan 2017 02:26:02 +0000 Subject: [PATCH 062/292] Make SQLITE_DIRECT_OVERFLOW_READ work for in WAL mode as long as the page being read is not in the the -wal file. FossilOrigin-Name: 9879be1899adf5634f551a2077b15ccb1133e4e3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 4 ++-- src/pager.c | 18 ++++++++++++------ src/pager.h | 6 ++++-- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 46dc00f8c4..133861bb31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sto\sthe\skvtest\sutility\sprogram.\s\sAdd\sthe\s--jmode\soption.\s\nImproved\soutput\sformatting. -D 2017-01-26T01:54:39.591 +C Make\sSQLITE_DIRECT_OVERFLOW_READ\swork\sfor\sin\sWAL\smode\sas\slong\sas\sthe\spage\nbeing\sread\sis\snot\sin\sthe\sthe\s-wal\sfile. +D 2017-01-26T02:26:02.936 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c f1b36bcfb1f9532ce64fe534153f11b8e2595d8b +F src/btree.c 4a4ae5fd82ee576b85163277610d8b8174831f16 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -377,8 +377,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 9dc72d23eebbdf992bd69f2ab954d0d3a27c7340 -F src/pager.h d1e944291030351f362a0a7da9b5c3e34e603e39 +F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b +F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa F src/parse.y 29153738a7322054359320eb00b5a4cd44389f20 F src/pcache.c 51070ec9b8251bbf9c6ea3d35fd96a458752929e F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50e60cb44fd3687dde5551d02bad60c323beaabc -R 6c49e01bee6ced9ad603e50cc8404852 +P 62a4851ccf88837d1c16dae8204f7f264e80e3c9 +R 5def997b16301f95d47461ef6a1c2e30 U drh -Z 5b3363ef063097392f2b7107d7fe03e8 +Z c8beae43adff961172e88276d51d2aaf diff --git a/manifest.uuid b/manifest.uuid index 7d6c267913..3045549ece 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62a4851ccf88837d1c16dae8204f7f264e80e3c9 \ No newline at end of file +9879be1899adf5634f551a2077b15ccb1133e4e3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b6bc66ea85..016e7f6cca 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4593,7 +4593,7 @@ static int accessPayload( ** 2) data is required from the start of this overflow page, and ** 3) the database is file-backed, and ** 4) there is no open write-transaction, and - ** 5) the database is not a WAL database, + ** 5) the page is not in the WAL file ** 6) all data from the page is being read. ** 7) at least 4 bytes have already been read into the output buffer ** @@ -4606,7 +4606,7 @@ static int accessPayload( && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && 0==sqlite3PagerUseWal(pBt->pPager) /* (5) */ + && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; diff --git a/src/pager.c b/src/pager.c index 5813b4db47..40c4dd9d88 100644 --- a/src/pager.c +++ b/src/pager.c @@ -814,14 +814,20 @@ static const unsigned char aJournalMagic[] = { #define isOpen(pFd) ((pFd)->pMethods!=0) /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. +** Return true if this pager uses a write-ahead log to read page pgno. +** Return false if the pager reads pgno directly from the database. */ -#ifndef SQLITE_OMIT_WAL -int sqlite3PagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ) +int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){ + u32 iRead = 0; + int rc; + if( pPager->pWal==0 ) return 0; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return rc || iRead; } -# define pagerUseWal(x) sqlite3PagerUseWal(x) +#endif +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 diff --git a/src/pager.h b/src/pager.h index dd57f598bd..585ef29497 100644 --- a/src/pager.h +++ b/src/pager.h @@ -178,14 +178,16 @@ int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); - int sqlite3PagerUseWal(Pager *pPager); +# ifdef SQLITE_DIRECT_OVERFLOW_READ + int sqlite3PagerUseWal(Pager *pPager, Pgno); +# endif # ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); int sqlite3PagerSnapshotRecover(Pager *pPager); # endif #else -# define sqlite3PagerUseWal(x) 0 +# define sqlite3PagerUseWal(x,y) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS From 8bb9fd3b457559e3dfee0b1fecf6ce7826de2636 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 16:27:32 +0000 Subject: [PATCH 063/292] Minor simplification and performance optimization for Direct Overflow Read. FossilOrigin-Name: 3e96d6efa867b765c8acf1454014b1e71b2e4f21 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 24 +++++++++--------------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 133861bb31..90d70de805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sSQLITE_DIRECT_OVERFLOW_READ\swork\sfor\sin\sWAL\smode\sas\slong\sas\sthe\spage\nbeing\sread\sis\snot\sin\sthe\sthe\s-wal\sfile. -D 2017-01-26T02:26:02.936 +C Minor\ssimplification\sand\sperformance\soptimization\sfor\sDirect\sOverflow\sRead. +D 2017-01-26T16:27:32.813 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 4a4ae5fd82ee576b85163277610d8b8174831f16 +F src/btree.c a168da7c4fff00dbd939c49ad4445d378eff1b11 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 62a4851ccf88837d1c16dae8204f7f264e80e3c9 -R 5def997b16301f95d47461ef6a1c2e30 +P 9879be1899adf5634f551a2077b15ccb1133e4e3 +R d7cd9a6ab86d59aef41165d7cf109b11 U drh -Z c8beae43adff961172e88276d51d2aaf +Z 50b1b915cab08ae10917e730aa3424e6 diff --git a/manifest.uuid b/manifest.uuid index 3045549ece..d3b805d95a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9879be1899adf5634f551a2077b15ccb1133e4e3 \ No newline at end of file +3e96d6efa867b765c8acf1454014b1e71b2e4f21 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 016e7f6cca..3150026a58 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4460,8 +4460,7 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - unsigned char * const pBufStart = pBuf; - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ #endif assert( pPage ); @@ -4472,9 +4471,6 @@ static int accessPayload( getCellInfo(pCur); aPayload = pCur->info.pPayload; -#ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = offset+amt==pCur->info.nPayload; -#endif assert( offset+amt <= pCur->info.nPayload ); assert( aPayload > pPage->aData ); @@ -4579,7 +4575,7 @@ static int accessPayload( ** range of data that is being read (eOp==0) or written (eOp!=0). */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; + sqlite3_file *fd; /* File from which to do direct overflow read */ #endif int a = amt; if( a + offset > ovflSize ){ @@ -4591,11 +4587,10 @@ static int accessPayload( ** ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) the database is file-backed, and - ** 4) there is no open write-transaction, and + ** 3) there is no open write-transaction, and + ** 4) the database is file-backed, and ** 5) the page is not in the WAL file - ** 6) all data from the page is being read. - ** 7) at least 4 bytes have already been read into the output buffer + ** 6) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -4603,15 +4598,14 @@ static int accessPayload( */ if( (eOp&0x01)==0 /* (1) */ && offset==0 /* (2) */ - && (bEnd || a==ovflSize) /* (6) */ - && pBt->inTransaction==TRANS_READ /* (4) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ + && pBt->inTransaction==TRANS_READ /* (3) */ + && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ - && &pBuf[-4]>=pBufStart /* (7) */ + && &pBuf[-4]>=pBufStart /* (6) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; - assert( aWrite>=pBufStart ); /* hence (7) */ + assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); From 83ec2761faa8e84345aa43927d2de73b0c96131f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 16:54:47 +0000 Subject: [PATCH 064/292] Performance optimization to sqlite3_blob_read(). FossilOrigin-Name: 7459f4b7ed4007d9ec44c3bf0fcba04f5f8540a9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 27 ++++++++++++++++++++------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 90d70de805..54fc47c9d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\ssimplification\sand\sperformance\soptimization\sfor\sDirect\sOverflow\sRead. -D 2017-01-26T16:27:32.813 +C Performance\soptimization\sto\ssqlite3_blob_read(). +D 2017-01-26T16:54:47.281 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c a168da7c4fff00dbd939c49ad4445d378eff1b11 +F src/btree.c 6fe6a5853148b623c4f5e288ae916180632ebbd3 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9879be1899adf5634f551a2077b15ccb1133e4e3 -R d7cd9a6ab86d59aef41165d7cf109b11 +P 3e96d6efa867b765c8acf1454014b1e71b2e4f21 +R 252dcd60be80c6c5c6ba84d2de14b6df U drh -Z 50b1b915cab08ae10917e730aa3424e6 +Z 39fbcc21c34e29a424df0af03b3f8d43 diff --git a/manifest.uuid b/manifest.uuid index d3b805d95a..051cfd0da2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e96d6efa867b765c8acf1454014b1e71b2e4f21 \ No newline at end of file +7459f4b7ed4007d9ec44c3bf0fcba04f5f8540a9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 3150026a58..0a2106e04b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4665,21 +4665,34 @@ int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); } + +/* +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. +*/ #ifndef SQLITE_OMIT_INCRBLOB -int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ int rc; if ( pCur->eState==CURSOR_INVALID ){ return SQLITE_ABORT; } assert( cursorOwnsBtShared(pCur) ); rc = restoreCursorPosition(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - rc = accessPayload(pCur, offset, amt, pBuf, 0); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); +} +int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); } - return rc; } #endif /* SQLITE_OMIT_INCRBLOB */ From d08406477afd21551ea6caadfb70c63a4dab23be Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 17:11:18 +0000 Subject: [PATCH 065/292] Performance optimization in sqlite3VdbeCloseStatement(). FossilOrigin-Name: 1e96e5ec1ee617cb5b5cbdc5a2ee79c8cc35821d --- manifest | 12 ++++---- manifest.uuid | 2 +- src/vdbeaux.c | 77 +++++++++++++++++++++++++-------------------------- 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 54fc47c9d1..11ce60cdc3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sto\ssqlite3_blob_read(). -D 2017-01-26T16:54:47.281 +C Performance\soptimization\sin\ssqlite3VdbeCloseStatement(). +D 2017-01-26T17:11:18.355 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -461,7 +461,7 @@ F src/vdbe.c 1fa3e8f3ee1cdfc56d9559bca9ca2c84e4a398be F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 -F src/vdbeaux.c 7c19b78999faae833e1be66dfa4e3deaf334d739 +F src/vdbeaux.c 6847b02aa2db536ed15d90f1fdc2923afef93c5b F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e96d6efa867b765c8acf1454014b1e71b2e4f21 -R 252dcd60be80c6c5c6ba84d2de14b6df +P 7459f4b7ed4007d9ec44c3bf0fcba04f5f8540a9 +R b420e83a1f23766d77baebce9b33f9ae U drh -Z 39fbcc21c34e29a424df0af03b3f8d43 +Z 2c16f151f6d1962e9b2a5835576e2c7c diff --git a/manifest.uuid b/manifest.uuid index 051cfd0da2..9f15fbe8cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7459f4b7ed4007d9ec44c3bf0fcba04f5f8540a9 \ No newline at end of file +1e96e5ec1ee617cb5b5cbdc5a2ee79c8cc35821d \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 44f2ad38f4..23c31d4029 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2497,60 +2497,59 @@ static void checkActiveVdbeCnt(sqlite3 *db){ ** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. ** Otherwise SQLITE_OK. */ -int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ +static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){ sqlite3 *const db = p->db; int rc = SQLITE_OK; + int i; + const int iSavepoint = p->iStatement-1; - /* If p->iStatement is greater than zero, then this Vdbe opened a - ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occurred, causing an emergency rollback. - ** In this case (db->nStatement==0), and there is nothing to do. - */ - if( db->nStatement && p->iStatement ){ - int i; - const int iSavepoint = p->iStatement-1; + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); - assert( db->nStatement>0 ); - assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - - for(i=0; inDb; i++){ - int rc2 = SQLITE_OK; - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( eOp==SAVEPOINT_ROLLBACK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); - } - if( rc2==SQLITE_OK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); - } - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - db->nStatement--; - p->iStatement = 0; - - if( rc==SQLITE_OK ){ + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ if( eOp==SAVEPOINT_ROLLBACK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); } if( rc==SQLITE_OK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + rc = rc2; } } + } + db->nStatement--; + p->iStatement = 0; - /* If the statement transaction is being rolled back, also restore the - ** database handles deferred constraint counter to the value it had when - ** the statement transaction was opened. */ + if( rc==SQLITE_OK ){ if( eOp==SAVEPOINT_ROLLBACK ){ - db->nDeferredCons = p->nStmtDefCons; - db->nDeferredImmCons = p->nStmtDefImmCons; + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + db->nDeferredImmCons = p->nStmtDefImmCons; } return rc; } +int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + if( p->db->nStatement && p->iStatement ){ + return vdbeCloseStatement(p, eOp); + } + return SQLITE_OK; +} + /* ** This function is called when a transaction opened by the database From fae58d51cee559af3c71860dce26ee0007f4780d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 17:26:44 +0000 Subject: [PATCH 066/292] Remove the obsolete lastRowid cache from the sqlite3VdbeExec() for a size reduction and performance improvement. FossilOrigin-Name: b4803184652e5f4f823c1521412bc480baeb3dbf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 11 +++-------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 11ce60cdc3..2bd3d3e394 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\ssqlite3VdbeCloseStatement(). -D 2017-01-26T17:11:18.355 +C Remove\sthe\sobsolete\slastRowid\scache\sfrom\sthe\ssqlite3VdbeExec()\sfor\sa\ssize\nreduction\sand\sperformance\simprovement. +D 2017-01-26T17:26:44.998 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -457,7 +457,7 @@ F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 1fa3e8f3ee1cdfc56d9559bca9ca2c84e4a398be +F src/vdbe.c 96f1639399feb97500d927219b87183aad3ef94c F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7459f4b7ed4007d9ec44c3bf0fcba04f5f8540a9 -R b420e83a1f23766d77baebce9b33f9ae +P 1e96e5ec1ee617cb5b5cbdc5a2ee79c8cc35821d +R f17134692e3127e9f97d9e12f2b91a46 U drh -Z 2c16f151f6d1962e9b2a5835576e2c7c +Z 7f3d9d2e9e1038bd56eb8c4623402887 diff --git a/manifest.uuid b/manifest.uuid index 9f15fbe8cb..313588dfbb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e96e5ec1ee617cb5b5cbdc5a2ee79c8cc35821d \ No newline at end of file +b4803184652e5f4f823c1521412bc480baeb3dbf \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6838acccd5..afb23007c6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -584,7 +584,6 @@ int sqlite3VdbeExec( Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ int *aPermute = 0; /* Permutation of columns for OP_Compare */ - i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ #endif @@ -960,7 +959,6 @@ case OP_Halt: { p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); pcx = sqlite3VdbeFrameRestore(pFrame); - lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ /* Instruction pcx is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt @@ -1682,9 +1680,7 @@ case OP_Function: { #endif MemSetTypeFlag(pCtx->pOut, MEM_Null); pCtx->fErrorOrAux = 0; - db->lastRowid = lastRowid; (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ /* If the function returned an error, throw an exception */ if( pCtx->fErrorOrAux ){ @@ -4418,7 +4414,7 @@ case OP_InsertInt: { #endif if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; if( pData->flags & MEM_Null ){ x.pData = 0; x.nData = 0; @@ -5863,7 +5859,7 @@ case OP_Program: { /* jump */ p->nFrame++; pFrame->pParent = p->pFrame; - pFrame->lastRowid = lastRowid; + pFrame->lastRowid = db->lastRowid; pFrame->nChange = p->nChange; pFrame->nDbChange = p->db->nChange; assert( pFrame->pAuxData==0 ); @@ -6804,7 +6800,7 @@ case OP_VUpdate: { sqlite3VtabImportErrmsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); - db->lastRowid = lastRowid = rowid; + db->lastRowid = rowid; } if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ @@ -7040,7 +7036,6 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - db->lastRowid = lastRowid; testcase( nVmStep>0 ); p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); From b7dab70a92e243559a97e6dbc7c81d930a6c46aa Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 18:00:00 +0000 Subject: [PATCH 067/292] Improvement to the OP_Permutation opcode to prevent it from using CPU cycles for initialization in prepared statements that do not use that opcode. FossilOrigin-Name: b4a98f65564a0d9fba2fef95ebd00a39b3e1e572 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 20 ++++++++++++++------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2bd3d3e394..f2f739774b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sobsolete\slastRowid\scache\sfrom\sthe\ssqlite3VdbeExec()\sfor\sa\ssize\nreduction\sand\sperformance\simprovement. -D 2017-01-26T17:26:44.998 +C Improvement\sto\sthe\sOP_Permutation\sopcode\sto\sprevent\sit\sfrom\susing\sCPU\scycles\nfor\sinitialization\sin\sprepared\sstatements\sthat\sdo\snot\suse\sthat\sopcode. +D 2017-01-26T18:00:00.328 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -457,7 +457,7 @@ F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 96f1639399feb97500d927219b87183aad3ef94c +F src/vdbe.c c27cc34be1d9169c1c191238025781684bdcd4ec F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e96e5ec1ee617cb5b5cbdc5a2ee79c8cc35821d -R f17134692e3127e9f97d9e12f2b91a46 +P b4803184652e5f4f823c1521412bc480baeb3dbf +R 5d2cc4b6c4ee841c772ca9230a25766d U drh -Z 7f3d9d2e9e1038bd56eb8c4623402887 +Z 39169c59805d0271c33a4d8ecd6430ca diff --git a/manifest.uuid b/manifest.uuid index 313588dfbb..bf55d47b01 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4803184652e5f4f823c1521412bc480baeb3dbf \ No newline at end of file +b4a98f65564a0d9fba2fef95ebd00a39b3e1e572 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index afb23007c6..ded4462398 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -583,7 +583,6 @@ int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ - int *aPermute = 0; /* Permutation of columns for OP_Compare */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ #endif @@ -2136,8 +2135,8 @@ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ /* Opcode: Permutation * * * P4 * ** -** Set the permutation used by the OP_Compare operator to be the array -** of integers in P4. +** Set the permutation used by the OP_Compare operator in the next +** instruction. The permutation is stored in the P4 operand. ** ** The permutation is only valid until the next OP_Compare that has ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should @@ -2149,7 +2148,8 @@ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai + 1; + assert( pOp[1].opcode==OP_Compare ); + assert( pOp[1].p5 & OPFLAG_PERMUTE ); break; } @@ -2182,8 +2182,17 @@ case OP_Compare: { int idx; CollSeq *pColl; /* Collating sequence to use on this term */ int bRev; /* True for DESCENDING sort order */ + int *aPermute; /* The permutation */ - if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0; + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ){ + aPermute = 0; + }else{ + assert( pOp>aOp ); + assert( pOp[-1].opcode==OP_Permutation ); + assert( pOp[-1].p4type==P4_INTARRAY ); + aPermute = pOp[-1].p4.ai + 1; + assert( aPermute!=0 ); + } n = pOp->p3; pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); @@ -2216,7 +2225,6 @@ case OP_Compare: { break; } } - aPermute = 0; break; } From 945b094632adb0bc835f4dacca84c7e3f7883dc6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Jan 2017 21:30:00 +0000 Subject: [PATCH 068/292] Remove an unreachable branch in the error handling logic for sqlite3BtreePayloadChecked(). FossilOrigin-Name: 293bf3ed7e40745349c83b202b27ed1b48517e1a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f2f739774b..1a7e283a21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvement\sto\sthe\sOP_Permutation\sopcode\sto\sprevent\sit\sfrom\susing\sCPU\scycles\nfor\sinitialization\sin\sprepared\sstatements\sthat\sdo\snot\suse\sthat\sopcode. -D 2017-01-26T18:00:00.328 +C Remove\san\sunreachable\sbranch\sin\sthe\serror\shandling\slogic\sfor\nsqlite3BtreePayloadChecked(). +D 2017-01-26T21:30:00.788 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 6fe6a5853148b623c4f5e288ae916180632ebbd3 +F src/btree.c bd72bb69abc7f3f3513308b9dd3749194b5d66d1 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4803184652e5f4f823c1521412bc480baeb3dbf -R 5d2cc4b6c4ee841c772ca9230a25766d +P b4a98f65564a0d9fba2fef95ebd00a39b3e1e572 +R 9abdd03cd61c16f0272c0fd74c72cba7 U drh -Z 39169c59805d0271c33a4d8ecd6430ca +Z 4b9455997c313589c1136c73cb07a07d diff --git a/manifest.uuid b/manifest.uuid index bf55d47b01..f992289f01 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4a98f65564a0d9fba2fef95ebd00a39b3e1e572 \ No newline at end of file +293bf3ed7e40745349c83b202b27ed1b48517e1a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0a2106e04b..e4a7aa160f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4683,7 +4683,7 @@ static SQLITE_NOINLINE int accessPayloadChecked( return SQLITE_ABORT; } assert( cursorOwnsBtShared(pCur) ); - rc = restoreCursorPosition(pCur); + rc = btreeRestoreCursorPosition(pCur); return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); } int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ From 42e28f12a31d38c22163b782dde53df195d894f3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 00:31:59 +0000 Subject: [PATCH 069/292] Simplify the accessPayload() routine so that it always populates the overflow page cache. In the one case where populating the page cache can lead to problems, simply invalidate the cache as soon as accessPayload() returns. This simplification reduces code size and helps accessPayload() to run a little faster. This backs out the eOp==2 mode of accessPayload() added by check-in [da59198505]. FossilOrigin-Name: 68e7a8c6765649195ef1ad9407d87d44a307b462 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 43 +++++++++++++++++-------------------------- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 1a7e283a21..0ab648517f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\serror\shandling\slogic\sfor\nsqlite3BtreePayloadChecked(). -D 2017-01-26T21:30:00.788 +C Simplify\sthe\saccessPayload()\sroutine\sso\sthat\sit\salways\spopulates\sthe\soverflow\npage\scache.\s\sIn\sthe\sone\scase\swhere\spopulating\sthe\spage\scache\scan\slead\sto\s\nproblems,\ssimply\sinvalidate\sthe\scache\sas\ssoon\sas\saccessPayload()\sreturns.\s\s\nThis\ssimplification\sreduces\scode\ssize\sand\shelps\saccessPayload()\sto\srun\sa\s\nlittle\sfaster.\s\sThis\sbacks\sout\sthe\seOp==2\smode\sof\saccessPayload()\sadded\sby\s\ncheck-in\s[da59198505]. +D 2017-01-27T00:31:59.289 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c bd72bb69abc7f3f3513308b9dd3749194b5d66d1 +F src/btree.c d42f290214e6615020b61866fe0747f02614964e F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4a98f65564a0d9fba2fef95ebd00a39b3e1e572 -R 9abdd03cd61c16f0272c0fd74c72cba7 +P 293bf3ed7e40745349c83b202b27ed1b48517e1a +R ec063431f8eb3294f488a586b69214cb U drh -Z 4b9455997c313589c1136c73cb07a07d +Z 81eb07f579988e3e7128daaaea36d064 diff --git a/manifest.uuid b/manifest.uuid index f992289f01..6d62a1cc6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -293bf3ed7e40745349c83b202b27ed1b48517e1a \ No newline at end of file +68e7a8c6765649195ef1ad9407d87d44a307b462 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e4a7aa160f..2575c774cf 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4424,7 +4424,6 @@ static int copyPayload( ** ** 0: The operation is a read. Populate the overflow cache. ** 1: The operation is a write. Populate the overflow cache. -** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -4432,13 +4431,13 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the current cursor entry uses one or more overflow pages and the -** eOp argument is not 2, this function may allocate space for and lazily -** populates the overflow page-list cache array (BtCursor.aOverflow). +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** -** Once an overflow page-list cache has been allocated, it may be +** Once an overflow page-list cache has been allocated, it must be ** invalidated if some other cursor writes to the same table, or if ** the cursor is moved to a different row. Additionally, in auto-vacuum ** mode, the following events may invalidate an overflow page-list cache. @@ -4464,10 +4463,10 @@ static int accessPayload( #endif assert( pPage ); + assert( eOp==0 || eOp==1 ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; @@ -4489,7 +4488,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -4505,14 +4504,13 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. - ** Except, do not allocate aOverflow[] for eOp==2. ** ** The aOverflow[] array is sized at one entry for each overflow page ** in the overflow chain. The page number of the first overflow page is ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array ** means "not yet known" (the cache is lazily populated). */ - if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ Pgno *aNew = (Pgno*)sqlite3Realloc( @@ -4533,9 +4531,7 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 - && pCur->aOverflow[offset/ovflSize] - ){ + if( pCur->aOverflow[offset/ovflSize] ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -4544,12 +4540,10 @@ static int accessPayload( assert( rc==SQLITE_OK && amt>0 ); while( nextPage ){ /* If required, populate the overflow page-list cache. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert( pCur->aOverflow[iIdx]==0 - || pCur->aOverflow[iIdx]==nextPage - || CORRUPT_DB ); - pCur->aOverflow[iIdx] = nextPage; - } + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -4557,11 +4551,7 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. - ** - ** Note that the aOverflow[] array must be allocated because eOp!=2 - ** here. If eOp==2, then offset==0 and this branch is never taken. */ - assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ @@ -4596,7 +4586,7 @@ static int accessPayload( ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( (eOp&0x01)==0 /* (1) */ + if( eOp==0 /* (1) */ && offset==0 /* (2) */ && pBt->inTransaction==TRANS_READ /* (3) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ @@ -4616,12 +4606,12 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, - ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) + (eOp==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } @@ -5253,7 +5243,8 @@ int sqlite3BtreeMovetoUnpacked( goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; From cdf360a0d5c3edfb49a44a7de9b39052f9562d2e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 01:13:49 +0000 Subject: [PATCH 070/292] Performance optimization in accessPayload(). FossilOrigin-Name: ebb1fd98d4e448aa6d8f1e5be7ddc5bedb3db95b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 0ab648517f..f9f99a3dde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\saccessPayload()\sroutine\sso\sthat\sit\salways\spopulates\sthe\soverflow\npage\scache.\s\sIn\sthe\sone\scase\swhere\spopulating\sthe\spage\scache\scan\slead\sto\s\nproblems,\ssimply\sinvalidate\sthe\scache\sas\ssoon\sas\saccessPayload()\sreturns.\s\s\nThis\ssimplification\sreduces\scode\ssize\sand\shelps\saccessPayload()\sto\srun\sa\s\nlittle\sfaster.\s\sThis\sbacks\sout\sthe\seOp==2\smode\sof\saccessPayload()\sadded\sby\s\ncheck-in\s[da59198505]. -D 2017-01-27T00:31:59.289 +C Performance\soptimization\sin\saccessPayload(). +D 2017-01-27T01:13:49.702 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c d42f290214e6615020b61866fe0747f02614964e +F src/btree.c 1329274c46bc554daf7acde825f72f76f66647ec F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 293bf3ed7e40745349c83b202b27ed1b48517e1a -R ec063431f8eb3294f488a586b69214cb +P 68e7a8c6765649195ef1ad9407d87d44a307b462 +R cab1deb289447016960f1fadc0c9e121 U drh -Z 81eb07f579988e3e7128daaaea36d064 +Z 955f6c694c099beee31ec142603efb52 diff --git a/manifest.uuid b/manifest.uuid index 6d62a1cc6d..a23f86a1ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68e7a8c6765649195ef1ad9407d87d44a307b462 \ No newline at end of file +ebb1fd98d4e448aa6d8f1e5be7ddc5bedb3db95b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2575c774cf..54664043d4 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4525,16 +4525,16 @@ static int accessPayload( } memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; - } - - /* If the overflow page-list cache has been allocated and the - ** entry for the first required overflow page is valid, skip - ** directly to it. - */ - if( pCur->aOverflow[offset/ovflSize] ){ - iIdx = (offset/ovflSize); - nextPage = pCur->aOverflow[iIdx]; - offset = (offset%ovflSize); + }else{ + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); + } } assert( rc==SQLITE_OK && amt>0 ); From 6ee610bfc021ade4f83261932180b9492fb639c8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 01:25:00 +0000 Subject: [PATCH 071/292] Another micro-optimization in accessPayload(). Slightly smaller and faster. FossilOrigin-Name: c012619b65d70b4ef6cf33532ef57d7f8ba42d74 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f9f99a3dde..f94e40935d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\saccessPayload(). -D 2017-01-27T01:13:49.702 +C Another\smicro-optimization\sin\saccessPayload().\s\sSlightly\ssmaller\sand\sfaster. +D 2017-01-27T01:25:00.940 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -333,7 +333,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1329274c46bc554daf7acde825f72f76f66647ec +F src/btree.c 6a63fa34e6fe86e87090e41963c0f2fcf9d3e16d F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68e7a8c6765649195ef1ad9407d87d44a307b462 -R cab1deb289447016960f1fadc0c9e121 +P ebb1fd98d4e448aa6d8f1e5be7ddc5bedb3db95b +R 63992a28f021aa7b8223daa601910fb3 U drh -Z 955f6c694c099beee31ec142603efb52 +Z 78e3ca04c2459249956256691b4d7091 diff --git a/manifest.uuid b/manifest.uuid index a23f86a1ed..23c39251fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebb1fd98d4e448aa6d8f1e5be7ddc5bedb3db95b \ No newline at end of file +c012619b65d70b4ef6cf33532ef57d7f8ba42d74 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 54664043d4..45a1c5a0f8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4617,16 +4617,16 @@ static int accessPayload( } } amt -= a; + if( amt==0 ) return rc; pBuf += a; } - if( amt==0 ) break; if( rc ) break; iIdx++; } } if( rc==SQLITE_OK && amt>0 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_BKPT; /* Overflow chain ends prematurely */ } return rc; } From 7aee83b935ccdd9031ffc2e4ab599cd8f5656981 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 01:52:42 +0000 Subject: [PATCH 072/292] In the command-line shell, enhance the ".mode" command so that it restores the default column and row separators for modes "line", "list", "column", and "tcl". FossilOrigin-Name: 58f02e6eae8fc9e2577fe435b0282fb46af3960d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f94e40935d..24e35fa9c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\smicro-optimization\sin\saccessPayload().\s\sSlightly\ssmaller\sand\sfaster. -D 2017-01-27T01:25:00.940 +C In\sthe\scommand-line\sshell,\senhance\sthe\s".mode"\scommand\sso\sthat\sit\srestores\sthe\ndefault\scolumn\sand\srow\sseparators\sfor\smodes\s"line",\s"list",\s"column",\sand\n"tcl". +D 2017-01-27T01:52:42.481 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -391,7 +391,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 -F src/shell.c 59de9acab4423a536277653f2a9dcdd1307989f3 +F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebb1fd98d4e448aa6d8f1e5be7ddc5bedb3db95b -R 63992a28f021aa7b8223daa601910fb3 +P c012619b65d70b4ef6cf33532ef57d7f8ba42d74 +R dc9b2fbf482ea9dbd0fd27947b6ee819 U drh -Z 78e3ca04c2459249956256691b4d7091 +Z 49761a5cfc1a2c7c58e7f0cde30087c6 diff --git a/manifest.uuid b/manifest.uuid index 23c39251fa..beee9d8c88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c012619b65d70b4ef6cf33532ef57d7f8ba42d74 \ No newline at end of file +58f02e6eae8fc9e2577fe435b0282fb46af3960d \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index db4362dd32..0e553d9fbb 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4288,15 +4288,20 @@ static int do_meta_command(char *zLine, ShellState *p){ int c2 = zMode[0]; if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ p->mode = MODE_Column; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); From 9ec82ff2ba27094ce75b0a3ea53063b72d803c13 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 13:14:12 +0000 Subject: [PATCH 073/292] Alternative ICU fix (compare to check-in [50e60cb4]) that avoids casting integers to pointers. FossilOrigin-Name: d9752c8f7c55426fd7d2b877c5cc3784f93b5349 --- ext/icu/icu.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/icu/icu.c b/ext/icu/icu.c index 50518d08a7..7c37812d86 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -500,7 +500,7 @@ int sqlite3IcuInit(sqlite3 *db){ unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"icu_load_collation", 2, SQLITE_UTF8, 255, icuLoadCollation}, + {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, @@ -521,7 +521,7 @@ int sqlite3IcuInit(sqlite3 *db){ const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( db, p->zName, p->nArg, p->enc, - p->iContext==255 ? (void*)db : (void*)(((char*)0)+p->iContext), + p->iContext ? (void*)db : (void*)0, p->xFunc, 0, 0 ); } diff --git a/manifest b/manifest index 24e35fa9c7..1c84aea007 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\senhance\sthe\s".mode"\scommand\sso\sthat\sit\srestores\sthe\ndefault\scolumn\sand\srow\sseparators\sfor\smodes\s"line",\s"list",\s"column",\sand\n"tcl". -D 2017-01-27T01:52:42.481 +C Alternative\sICU\sfix\s(compare\sto\scheck-in\s[50e60cb4])\sthat\savoids\scasting\nintegers\sto\spointers. +D 2017-01-27T13:14:12.467 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -202,7 +202,7 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 -F ext/icu/icu.c bae1dde13091b425bbed119a09a11aee14644764 +F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c012619b65d70b4ef6cf33532ef57d7f8ba42d74 -R dc9b2fbf482ea9dbd0fd27947b6ee819 +P 58f02e6eae8fc9e2577fe435b0282fb46af3960d +R 4a4a07080dc840a599a4cd01566373f4 U drh -Z 49761a5cfc1a2c7c58e7f0cde30087c6 +Z 816a99767333ba2de388be8cc7b0ed87 diff --git a/manifest.uuid b/manifest.uuid index beee9d8c88..92d5c8ac0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58f02e6eae8fc9e2577fe435b0282fb46af3960d \ No newline at end of file +d9752c8f7c55426fd7d2b877c5cc3784f93b5349 \ No newline at end of file From 50133dea285803571214c3459f1ce4d79eea25ec Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 27 Jan 2017 17:02:26 +0000 Subject: [PATCH 074/292] Fix a problem causing the pre-update hook to be invoked by DROP TABLE statements. FossilOrigin-Name: fbb6bf1b69cfd581b4ffd778c344e3fbd9c9406f --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/delete.c | 4 +++- test/hook.test | 10 ++++++---- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 1c84aea007..55464d8c36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Alternative\sICU\sfix\s(compare\sto\scheck-in\s[50e60cb4])\sthat\savoids\scasting\nintegers\sto\spointers. -D 2017-01-27T13:14:12.467 +C Fix\sa\sproblem\scausing\sthe\spre-update\shook\sto\sbe\sinvoked\sby\sDROP\sTABLE\nstatements. +D 2017-01-27T17:02:26.115 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -342,7 +342,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c 8a444fea8340989d6b1be2e122c55bfc61ce69be +F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae @@ -832,7 +832,7 @@ F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test f6a48d33817f0ca1a39a4d6605fe7e9da8077522 +F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58f02e6eae8fc9e2577fe435b0282fb46af3960d -R 4a4a07080dc840a599a4cd01566373f4 -U drh -Z 816a99767333ba2de388be8cc7b0ed87 +P d9752c8f7c55426fd7d2b877c5cc3784f93b5349 +R 7c4166e1e335b0337f6f8b4996b52965 +U dan +Z 241b65c68385a1329b03731baa61f118 diff --git a/manifest.uuid b/manifest.uuid index 92d5c8ac0b..e1bf32dbf9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9752c8f7c55426fd7d2b877c5cc3784f93b5349 \ No newline at end of file +fbb6bf1b69cfd581b4ffd778c344e3fbd9c9406f \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 1d31622237..0683f9b9dd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -715,7 +715,9 @@ void sqlite3GenerateRowDelete( u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + if( pParse->nested==0 ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + } if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } diff --git a/test/hook.test b/test/hook.test index 0c24ec7313..9ba220cded 100644 --- a/test/hook.test +++ b/test/hook.test @@ -849,10 +849,12 @@ do_preupdate_test 7.6.4 { } # No preupdate callbacks for modifying sqlite_master. -do_preupdate_test 8.1 { - CREATE TABLE x1(x, y); -} { -} +do_preupdate_test 8.1 { CREATE TABLE x1(x, y); } { } +do_preupdate_test 8.2 { ALTER TABLE x1 ADD COLUMN z } { } +do_preupdate_test 8.3 { ALTER TABLE x1 RENAME TO y1 } { } +do_preupdate_test 8.4 { CREATE INDEX y1x ON y1(x) } { } +do_preupdate_test 8.5 { CREATE VIEW v1 AS SELECT * FROM y1 } { } +do_preupdate_test 8.6 { DROP TABLE y1 } { } #------------------------------------------------------------------------- reset_db From cc04dacb8545a29a7f22a228277c69dbbe8628ba Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Jan 2017 19:27:38 +0000 Subject: [PATCH 075/292] Add the sha1.c loadable extension that implements the sha1() and sha1_query() SQL functions. FossilOrigin-Name: 24e77c1cef163a9822635570b3211789ff23d5c9 --- ext/misc/sha1.c | 399 ++++++++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +- manifest.uuid | 2 +- 3 files changed, 407 insertions(+), 7 deletions(-) create mode 100644 ext/misc/sha1.c diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c new file mode 100644 index 0000000000..dbf15a95ef --- /dev/null +++ b/ext/misc/sha1.c @@ -0,0 +1,399 @@ +/* +** 2017-01-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha1(X) +** sha1_query(Y) +** +** The sha1(X) function computes the SHA1 hash of the input X, or NULL if +** X is NULL. +** +** The sha1_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/****************************************************************************** +** The Hash Engine +*/ +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + + +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut /* Store hexadecimal hash here */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha1(X) function. +** +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hash without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + char zOut[44]; + + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[8]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n); + hash_step(&cx, z, n); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n); + hash_step(&cx, z, n); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8, 0, + sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8, 0, + sha1QueryFunc, 0, 0); + } + return rc; +} diff --git a/manifest b/manifest index 55464d8c36..5468756e11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\sthe\spre-update\shook\sto\sbe\sinvoked\sby\sDROP\sTABLE\nstatements. -D 2017-01-27T17:02:26.115 +C Add\sthe\ssha1.c\sloadable\sextension\sthat\simplements\sthe\ssha1()\sand\ssha1_query()\nSQL\sfunctions. +D 2017-01-27T19:27:38.700 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -222,6 +222,7 @@ F ext/misc/remember.c 8440f8d0b452c5cdefb62b57135ccd1267aa729d F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 +F ext/misc/sha1.c b2e4eb8e26f09701ec15548395baf698f00e5895 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1547,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9752c8f7c55426fd7d2b877c5cc3784f93b5349 -R 7c4166e1e335b0337f6f8b4996b52965 -U dan -Z 241b65c68385a1329b03731baa61f118 +P fbb6bf1b69cfd581b4ffd778c344e3fbd9c9406f +R 389e8cfe151c91b893dde5f7934ffeac +U drh +Z 4597f51eb6c4ecdd76ade7480dc74dac diff --git a/manifest.uuid b/manifest.uuid index e1bf32dbf9..4866d1d1f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fbb6bf1b69cfd581b4ffd778c344e3fbd9c9406f \ No newline at end of file +24e77c1cef163a9822635570b3211789ff23d5c9 \ No newline at end of file From 2d2e4f389584fec0b27100107b18bb835ddeba94 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Jan 2017 06:50:15 +0000 Subject: [PATCH 076/292] Fix a typo in the docs for sqlite3_update_hook(). FossilOrigin-Name: 7c029655cc3369a1e46741fdd3701d36d420b28b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5468756e11..41ba2e4d2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssha1.c\sloadable\sextension\sthat\simplements\sthe\ssha1()\sand\ssha1_query()\nSQL\sfunctions. -D 2017-01-27T19:27:38.700 +C Fix\sa\stypo\sin\sthe\sdocs\sfor\ssqlite3_update_hook(). +D 2017-01-28T06:50:15.642 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -393,7 +393,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 -F src/sqlite.h.in 1971ab9709e010d52a02a1a6276d5a2f9b947476 +F src/sqlite.h.in eefd49aa3aae3b4acd7d7e2f8f9b0db8ae0aff3b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h 341ce9e5b0397771fa6bd9dadb8ef4cbbd6224d0 @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fbb6bf1b69cfd581b4ffd778c344e3fbd9c9406f -R 389e8cfe151c91b893dde5f7934ffeac -U drh -Z 4597f51eb6c4ecdd76ade7480dc74dac +P 24e77c1cef163a9822635570b3211789ff23d5c9 +R f1c881d122d9ce415eab840d4253340a +U dan +Z 83c68a9a3025b00f2574a4c70122bf9b diff --git a/manifest.uuid b/manifest.uuid index 4866d1d1f8..140867e588 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24e77c1cef163a9822635570b3211789ff23d5c9 \ No newline at end of file +7c029655cc3369a1e46741fdd3701d36d420b28b \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 87d5d121cb..0bf386f72a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5417,7 +5417,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook -** is not invoked when duplication rows are deleted because of an +** is not invoked when conflicting rows are deleted because of an ** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook ** invoked when rows are deleted using the [truncate optimization]. ** The exceptions defined in this paragraph might change in a future From 27c846773497d27eb6f04f7f7b03d073b99b319d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Jan 2017 13:40:55 +0000 Subject: [PATCH 077/292] In the speed-check.sh script, automatically invoke "fossil test-diff --tk" on the cachegrind output against trunk, if not generating a trunk run. FossilOrigin-Name: aa1ab37100a91ab4bb91d50a1267c26967efcb21 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/speed-check.sh | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 41ba2e4d2c..3e4c3fcfe4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sdocs\sfor\ssqlite3_update_hook(). -D 2017-01-28T06:50:15.642 +C In\sthe\sspeed-check.sh\sscript,\sautomatically\sinvoke\s"fossil\stest-diff\s--tk"\son\nthe\scachegrind\soutput\sagainst\strunk,\sif\snot\sgenerating\sa\strunk\srun. +D 2017-01-28T13:40:55.453 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -1508,7 +1508,7 @@ F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a -F tool/speed-check.sh 65ac2f5b00771b9dcefb95bebae1aab76c537ea3 +F tool/speed-check.sh 9630ba0468b609c52f48309243d4eb6e9c34deda F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 24e77c1cef163a9822635570b3211789ff23d5c9 -R f1c881d122d9ce415eab840d4253340a -U dan -Z 83c68a9a3025b00f2574a4c70122bf9b +P 7c029655cc3369a1e46741fdd3701d36d420b28b +R e044c4e027113b10098b21316c3901f3 +U drh +Z 82ae1e57a20d4dbe9187373cc6493ae1 diff --git a/manifest.uuid b/manifest.uuid index 140867e588..fc7ceb70f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c029655cc3369a1e46741fdd3701d36d420b28b \ No newline at end of file +aa1ab37100a91ab4bb91d50a1267c26967efcb21 \ No newline at end of file diff --git a/tool/speed-check.sh b/tool/speed-check.sh index c6f587e177..1e1389f2fd 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -142,3 +142,6 @@ fi if test $doExplain -eq 1; then ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt fi +if test "$NAME" != "trunk"; then + fossil test-diff --tk cout-trunk.txt cout-$NAME.txt +fi From 6034d476186cda5b18f6003b8c3f66cb0e216d16 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Jan 2017 15:26:14 +0000 Subject: [PATCH 078/292] Updates to the sqlite3_blob documentation. No changes to code. FossilOrigin-Name: 426b440a5745f9c431c6a3d9ba542af61a6a83fb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3e4c3fcfe4..6c861e5ee0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sspeed-check.sh\sscript,\sautomatically\sinvoke\s"fossil\stest-diff\s--tk"\son\nthe\scachegrind\soutput\sagainst\strunk,\sif\snot\sgenerating\sa\strunk\srun. -D 2017-01-28T13:40:55.453 +C Updates\sto\sthe\ssqlite3_blob\sdocumentation.\s\sNo\schanges\sto\scode. +D 2017-01-28T15:26:14.397 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -393,7 +393,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 -F src/sqlite.h.in eefd49aa3aae3b4acd7d7e2f8f9b0db8ae0aff3b +F src/sqlite.h.in a14cd1f946b9565c4d801c4adeeaa4543fbd6eb4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h 341ce9e5b0397771fa6bd9dadb8ef4cbbd6224d0 @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c029655cc3369a1e46741fdd3701d36d420b28b -R e044c4e027113b10098b21316c3901f3 +P aa1ab37100a91ab4bb91d50a1267c26967efcb21 +R e94128da2458d76e1f9232d83fbb5c60 U drh -Z 82ae1e57a20d4dbe9187373cc6493ae1 +Z 85ea4b53c0f0f74a70744dcf483d3a5b diff --git a/manifest.uuid b/manifest.uuid index fc7ceb70f4..6d893bf0fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa1ab37100a91ab4bb91d50a1267c26967efcb21 \ No newline at end of file +426b440a5745f9c431c6a3d9ba542af61a6a83fb \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0bf386f72a..d80de6d519 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6199,6 +6199,12 @@ typedef struct sqlite3_blob sqlite3_blob; ** [database connection] error code and message accessible via ** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** +** A BLOB referenced by sqlite3_blob_open() and be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed once after the [BLOB handle] is opened. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -6222,6 +6228,10 @@ typedef struct sqlite3_blob sqlite3_blob; ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. */ int sqlite3_blob_open( sqlite3*, @@ -6237,11 +6247,11 @@ int sqlite3_blob_open( ** CAPI3REF: Move a BLOB Handle to a New Row ** METHOD: sqlite3_blob ** -** ^This function is used to move an existing blob handle so that it points +** ^This function is used to move an existing [BLOB handle] so that it points ** to a different row of the same database table. ^The new row is identified ** by the rowid value passed as the second argument. Only the row can be ** changed. ^The database, table and column on which the blob handle is open -** remain the same. Moving an existing blob handle to a new row can be +** remain the same. Moving an existing [BLOB handle] to a new row is ** faster than closing the existing handle and opening a new one. ** ** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - From f64ece143c4d71a5d8f8a67cd643cd0ef683c29e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Jan 2017 19:45:34 +0000 Subject: [PATCH 079/292] Avoid redundant table b-tree cursor seeks in UPDATE statements that use the two-pass strategy. FossilOrigin-Name: dc555b1039c6930f6d15355c698ff917a85e8056 --- ext/session/session1.test | 19 +++++++++++++++++++ manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- src/update.c | 2 +- src/wherecode.c | 5 ++++- test/update2.test | 25 +++++++++++++++++++++++++ 7 files changed, 62 insertions(+), 15 deletions(-) diff --git a/ext/session/session1.test b/ext/session/session1.test index 09d35af7fc..5216824bdd 100644 --- a/ext/session/session1.test +++ b/ext/session/session1.test @@ -582,4 +582,23 @@ do_iterator_test 11.2 * { } { } +reset_db +do_execsql_test 12.1 { + CREATE TABLE t1(r INTEGER PRIMARY KEY, a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 1, 2); + INSERT INTO t1 VALUES(3, 1, 3); +} + +do_iterator_test 12.2 * { + UPDATE t1 SET b='one' WHERE a=1; +} { + {UPDATE t1 0 X.. {i 1 {} {} i 1} {{} {} {} {} t one}} + {UPDATE t1 0 X.. {i 2 {} {} i 2} {{} {} {} {} t one}} + {UPDATE t1 0 X.. {i 3 {} {} i 3} {{} {} {} {} t one}} +} + + + finish_test diff --git a/manifest b/manifest index 6c861e5ee0..ddbf2ea5df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\ssqlite3_blob\sdocumentation.\s\sNo\schanges\sto\scode. -D 2017-01-28T15:26:14.397 +C Avoid\sredundant\stable\sb-tree\scursor\sseeks\sin\sUPDATE\sstatements\sthat\suse\sthe\ntwo-pass\sstrategy. +D 2017-01-28T19:45:34.617 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -288,7 +288,7 @@ F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a -F ext/session/session1.test e5125b216d1e8c91e0984b361b0b68529e7c5dfb +F ext/session/session1.test 482e9b861169d0b7e283e1563a28c0fa32779fa5 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 @@ -396,7 +396,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in a14cd1f946b9565c4d801c4adeeaa4543fbd6eb4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 341ce9e5b0397771fa6bd9dadb8ef4cbbd6224d0 +F src/sqliteInt.h c246370f9d8de3af36052d7860cd35c0e2daea6a F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -454,7 +454,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 -F src/update.c b356b29d04c71f33c779f2cb557cf953819bdd7a +F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 @@ -474,7 +474,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c bc71775e23d23334e8f449aa31012d692dc09cb2 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d -F src/wherecode.c e04ac8f24c3ac8621df6c3be3ac8c7d4fa893745 +F src/wherecode.c 99a8ced164c75edf41b3a865a75381c9adb38b28 F src/whereexpr.c 35ad025389a632a3987a35617c878be3b3d70dc6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1355,7 +1355,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 -F test/update2.test 7f72ef307d58be09a8ef61db23a7ef60e5af019d +F test/update2.test 81772e5c9b93d1006d66d65e9a1327281bdec4e8 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa1ab37100a91ab4bb91d50a1267c26967efcb21 -R e94128da2458d76e1f9232d83fbb5c60 -U drh -Z 85ea4b53c0f0f74a70744dcf483d3a5b +P 426b440a5745f9c431c6a3d9ba542af61a6a83fb +R cb9f34739a33405b5f0617b1ee3426fc +U dan +Z 2ac8b1778a518257277e1132fa6dad3a diff --git a/manifest.uuid b/manifest.uuid index 6d893bf0fe..0e49c05605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -426b440a5745f9c431c6a3d9ba542af61a6a83fb \ No newline at end of file +dc555b1039c6930f6d15355c698ff917a85e8056 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7ea8748b73..3f0da18e9c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2592,7 +2592,7 @@ struct SrcList { #define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ #define WHERE_SEEK_TABLE 0x0400 /* Do not defer seeks on main table */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ - /* 0x1000 not currently used */ +#define WHERE_SEEK_UNIQ_TABLE 0x1000 /* Do not defer seeks if unique */ /* 0x2000 not currently used */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ diff --git a/src/update.c b/src/update.c index 45af001c3e..8fca7eb988 100644 --- a/src/update.c +++ b/src/update.c @@ -392,7 +392,7 @@ void sqlite3Update( ** be deleted as a result of REPLACE conflict handling. Any of these ** things might disturb a cursor being used to scan through the table ** or index, causing a single-pass approach to malfunction. */ - flags = WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE; + flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE; if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ flags |= WHERE_ONEPASS_MULTIROW; } diff --git a/src/wherecode.c b/src/wherecode.c index 032c56fef3..58e040628a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1591,7 +1591,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){ + if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE) || ( + (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE) + && (pWInfo->eOnePass==ONEPASS_SINGLE) + )){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); diff --git a/test/update2.test b/test/update2.test index 61ac9792da..6a359d11a9 100644 --- a/test/update2.test +++ b/test/update2.test @@ -176,5 +176,30 @@ foreach {tn sql} { } } +#------------------------------------------------------------------------- +# +do_execsql_test 5.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX x1c ON x1(b, c); + INSERT INTO x1 VALUES(1, 'a', 1); + INSERT INTO x1 VALUES(2, 'a', 2); + INSERT INTO x1 VALUES(3, 'a', 3); +} + +do_execsql_test 5.1.1 { + UPDATE x1 SET c=c+1 WHERE b='a'; +} + +do_execsql_test 5.1.2 { + SELECT * FROM x1; +} {1 a 2 2 a 3 3 a 4} + +do_test 5.2 { + catch { array unset A } + db eval { EXPLAIN UPDATE x1 SET c=c+1 WHERE b='a' } { incr A($opcode) } + set A(NotExists) +} {1} + + finish_test From 51b15c3871a264dc4867eeda97faed19f86bdb2b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 28 Jan 2017 19:53:51 +0000 Subject: [PATCH 080/292] Fix a couple comment typos. No changes to code. FossilOrigin-Name: 2a2e7d86b275c970726f642a37a098950a3b906e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ddbf2ea5df..02a68de80a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredundant\stable\sb-tree\scursor\sseeks\sin\sUPDATE\sstatements\sthat\suse\sthe\ntwo-pass\sstrategy. -D 2017-01-28T19:45:34.617 +C Fix\sa\scouple\scomment\stypos.\s\sNo\schanges\sto\scode. +D 2017-01-28T19:53:51.613 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -393,7 +393,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 -F src/sqlite.h.in a14cd1f946b9565c4d801c4adeeaa4543fbd6eb4 +F src/sqlite.h.in 8fd2b1a4e4aac023d4533313442528b81105fab3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h c246370f9d8de3af36052d7860cd35c0e2daea6a @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 426b440a5745f9c431c6a3d9ba542af61a6a83fb -R cb9f34739a33405b5f0617b1ee3426fc -U dan -Z 2ac8b1778a518257277e1132fa6dad3a +P dc555b1039c6930f6d15355c698ff917a85e8056 +R 85ad9229fc7d18817a896fd231c69045 +U mistachkin +Z 8ca9913e082eb1ea0368f4a7e44ff324 diff --git a/manifest.uuid b/manifest.uuid index 0e49c05605..d337122ecd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc555b1039c6930f6d15355c698ff917a85e8056 \ No newline at end of file +2a2e7d86b275c970726f642a37a098950a3b906e \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d80de6d519..b0ec7f7e15 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6199,12 +6199,12 @@ typedef struct sqlite3_blob sqlite3_blob; ** [database connection] error code and message accessible via ** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** -** A BLOB referenced by sqlite3_blob_open() and be read using the +** A BLOB referenced by sqlite3_blob_open() may be read using the ** [sqlite3_blob_read()] interface and modified by using ** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a ** different row of the same table using the [sqlite3_blob_reopen()] ** interface. However, the column, table, or database of a [BLOB handle] -** cannot be changed once after the [BLOB handle] is opened. +** cannot be changed after the [BLOB handle] is opened. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects From d26cc54183849acb2f53537e67e5968ada4619d9 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Jan 2017 20:46:37 +0000 Subject: [PATCH 081/292] In the amalgamation, allocate the parser engine object from stack rather than from heap, for improved performance. This only happens in the amalgamation, since otherwise the sqlite3RunParser() routine does not know the object size. FossilOrigin-Name: 4fe879d4b5da6ae0688a7a99004683a234966597 --- manifest | 20 +++++++------- manifest.uuid | 2 +- src/parse.y | 13 +++++++++ src/sqliteInt.h | 6 +++-- src/tokenize.c | 12 +++++++++ tool/lempar.c | 71 ++++++++++++++++++++++++++++++------------------- 6 files changed, 84 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 02a68de80a..332ad282a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\scomment\stypos.\s\sNo\schanges\sto\scode. -D 2017-01-28T19:53:51.613 +C In\sthe\samalgamation,\sallocate\sthe\sparser\sengine\sobject\sfrom\sstack\srather\sthan\nfrom\sheap,\sfor\simproved\sperformance.\s\sThis\sonly\shappens\sin\sthe\samalgamation,\nsince\sotherwise\sthe\ssqlite3RunParser()\sroutine\sdoes\snot\sknow\sthe\sobject\ssize. +D 2017-01-28T20:46:37.958 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -380,7 +380,7 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y 29153738a7322054359320eb00b5a4cd44389f20 +F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 51070ec9b8251bbf9c6ea3d35fd96a458752929e F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 @@ -396,7 +396,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 8fd2b1a4e4aac023d4533313442528b81105fab3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h c246370f9d8de3af36052d7860cd35c0e2daea6a +F src/sqliteInt.h fd6815792077b7dc69f5167117cc5727fe95ed5a F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -451,7 +451,7 @@ F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 5c2f516876fc27fbd7753913f032f49eb89e83b5 +F src/tokenize.c 25ccc63ae2c4163933221f3181c9982b47cd08d2 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 @@ -1476,7 +1476,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7 -F tool/lempar.c 320d630b44da693407684c64d9fa91a163419dac +F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc555b1039c6930f6d15355c698ff917a85e8056 -R 85ad9229fc7d18817a896fd231c69045 -U mistachkin -Z 8ca9913e082eb1ea0368f4a7e44ff324 +P 2a2e7d86b275c970726f642a37a098950a3b906e +R 9e84d1f95c544016e0312212ee6f639c +U drh +Z 9df531d71c0beb5e9e8a394dca7276a3 diff --git a/manifest.uuid b/manifest.uuid index d337122ecd..87236c5ed3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a2e7d86b275c970726f642a37a098950a3b906e \ No newline at end of file +4fe879d4b5da6ae0688a7a99004683a234966597 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index c9dbc767cd..9cada2a1be 100644 --- a/src/parse.y +++ b/src/parse.y @@ -65,6 +65,19 @@ */ #define YYPARSEFREENEVERNULL 1 +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + /* ** Alternative datatype for the argument to the malloc() routine passed ** into sqlite3ParserAlloc(). The default is size_t. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3f0da18e9c..4b7bdf5937 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4073,8 +4073,10 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); /* ** The interface to the LEMON-generated parser */ -void *sqlite3ParserAlloc(void*(*)(u64)); -void sqlite3ParserFree(void*, void(*)(void*)); +#ifndef SQLITE_AMALGAMATION + void *sqlite3ParserAlloc(void*(*)(u64)); + void sqlite3ParserFree(void*, void(*)(void*)); +#endif void sqlite3Parser(void*, int, Token, Parse*); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); diff --git a/src/tokenize.c b/src/tokenize.c index c400dcd554..7bb1179e20 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -481,6 +481,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + unsigned char zSpace[sizeof(yyParser)]; /* Space for parser engine object */ +#endif assert( zSql!=0 ); mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -492,11 +495,16 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ i = 0; assert( pzErrMsg!=0 ); /* sqlite3ParserTrace(stdout, "parser: "); */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + pEngine = zSpace; + sqlite3ParserInit(pEngine); +#else pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ sqlite3OomFault(db); return SQLITE_NOMEM_BKPT; } +#endif assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); @@ -548,7 +556,11 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ ); sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + sqlite3ParserFinalize(pEngine); +#else sqlite3ParserFree(pEngine, sqlite3_free); +#endif if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM_BKPT; } diff --git a/tool/lempar.c b/tool/lempar.c index dab8343f3a..21ed9d79ab 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -316,6 +316,31 @@ static int yyGrowStack(yyParser *p){ # define YYMALLOCARGTYPE size_t #endif +/* Initialize a new parser that has already been allocated. +*/ +void ParseInit(void *yypParser){ + yyParser *pParser = (yyParser*)yypParser; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yytos = NULL; + pParser->yystack = NULL; + pParser->yystksz = 0; + if( yyGrowStack(pParser) ){ + pParser->yystack = &pParser->yystk0; + pParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + pParser->yyerrcnt = -1; +#endif + pParser->yytos = pParser->yystack; + pParser->yystack[0].stateno = 0; + pParser->yystack[0].major = 0; +} + +#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -331,28 +356,11 @@ static int yyGrowStack(yyParser *p){ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ){ -#ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; -#endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; - } + if( pParser ) ParseInit(pParser); return pParser; } +#endif /* Parse_ENGINEALWAYSONSTACK */ + /* The following function deletes the "minor type" or semantic value ** associated with a symbol. The symbol can be either a terminal @@ -406,6 +414,18 @@ static void yy_pop_parser_stack(yyParser *pParser){ yy_destructor(pParser, yytos->major, &yytos->minor); } +/* +** Clear all secondary memory allocations from the parser +*/ +void ParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef Parse_ENGINEALWAYSONSTACK /* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. @@ -418,16 +438,13 @@ void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ - yyParser *pParser = (yyParser*)p; #ifndef YYPARSEFREENEVERNULL - if( pParser==0 ) return; + if( p==0 ) return; #endif - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif - (*freeProc)((void*)pParser); + ParseFinalize(p); + (*freeProc)(p); } +#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. From e7eeeb99f7827fc2838cf730adfb58f5419a41e7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Jan 2017 11:38:19 +0000 Subject: [PATCH 082/292] Fix building with SQLITE_OMIT_FOREIGN_KEY defined. FossilOrigin-Name: e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 332ad282a9..229623bcc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\samalgamation,\sallocate\sthe\sparser\sengine\sobject\sfrom\sstack\srather\sthan\nfrom\sheap,\sfor\simproved\sperformance.\s\sThis\sonly\shappens\sin\sthe\samalgamation,\nsince\sotherwise\sthe\ssqlite3RunParser()\sroutine\sdoes\snot\sknow\sthe\sobject\ssize. -D 2017-01-28T20:46:37.958 +C Fix\sbuilding\swith\sSQLITE_OMIT_FOREIGN_KEY\sdefined. +D 2017-01-30T11:38:19.468 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -396,7 +396,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 8fd2b1a4e4aac023d4533313442528b81105fab3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h fd6815792077b7dc69f5167117cc5727fe95ed5a +F src/sqliteInt.h 6f29b23415ce207c2fda61054bf244b74f96359c F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2a2e7d86b275c970726f642a37a098950a3b906e -R 9e84d1f95c544016e0312212ee6f639c -U drh -Z 9df531d71c0beb5e9e8a394dca7276a3 +P 4fe879d4b5da6ae0688a7a99004683a234966597 +R c0ed5c24fdff676195cd77401e89700b +U dan +Z 6d1b138d17ccacee0c622f2ed331ca89 diff --git a/manifest.uuid b/manifest.uuid index 87236c5ed3..eec33fc0a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fe879d4b5da6ae0688a7a99004683a234966597 \ No newline at end of file +e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4b7bdf5937..4a28ebcd5a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4186,6 +4186,7 @@ const char *sqlite3JournalModename(int); #define sqlite3FkDropTable(a,b,c) #define sqlite3FkOldmask(a,b) 0 #define sqlite3FkRequired(a,b,c,d) 0 + #define sqlite3FkReferences(a) 0 #endif #ifndef SQLITE_OMIT_FOREIGN_KEY void sqlite3FkDelete(sqlite3 *, Table*); From cb9a364390f2db2bb9340045e7973d3f639110bb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Jan 2017 19:44:53 +0000 Subject: [PATCH 083/292] Experimental change to invoke the preupdate hook when WITHOUT ROWID tables are written. FossilOrigin-Name: 856f8604c59c8fdd9bfb7d86fc0e212f091ab49a --- ext/session/session1.test | 245 +++++++++++++++++++----------------- ext/session/sessionwor.test | 57 +++++++++ manifest | 27 ++-- manifest.uuid | 2 +- src/insert.c | 17 ++- src/vdbe.c | 13 +- src/vdbeInt.h | 1 + src/vdbeapi.c | 6 + src/vdbeaux.c | 11 +- test/hook2.test | 218 ++++++++++++++++++++++++++++++++ 10 files changed, 458 insertions(+), 139 deletions(-) create mode 100644 ext/session/sessionwor.test create mode 100644 test/hook2.test diff --git a/ext/session/session1.test b/ext/session/session1.test index 5216824bdd..c96c248c5e 100644 --- a/ext/session/session1.test +++ b/ext/session/session1.test @@ -20,46 +20,61 @@ ifcapable !session {finish_test; return} set testprefix session1 -do_execsql_test 1.0 { - CREATE TABLE t1(x PRIMARY KEY, y); +# Run all tests in this file twice. Once with "WITHOUT ROWID", and once +# with regular rowid tables. +# +foreach {tn trailing} { + 1 "" + 2 " WITHOUT ROWID " +} { +eval [string map [list %WR% $trailing] { + +db close +forcedelete test.db test.db2 +reset_db + +do_execsql_test $tn.1.0 { + CREATE TABLE t1(x PRIMARY KEY, y) %WR%; INSERT INTO t1 VALUES('abc', 'def'); } #------------------------------------------------------------------------- # Test creating, attaching tables to and deleting session objects. # -do_test 1.1 { sqlite3session S db main } {S} -do_test 1.2 { S delete } {} -do_test 1.3 { sqlite3session S db main } {S} -do_test 1.4 { S attach t1 } {} -do_test 1.5 { S delete } {} -do_test 1.6 { sqlite3session S db main } {S} -do_test 1.7 { S attach t1 ; S attach t2 ; S attach t3 } {} -do_test 1.8 { S attach t1 ; S attach t2 ; S attach t3 } {} -do_test 1.9 { S delete } {} -do_test 1.10 { +do_test $tn.1.1 { sqlite3session S db main } {S} +do_test $tn.1.2 { S delete } {} +do_test $tn.1.3 { sqlite3session S db main } {S} +do_test $tn.1.4 { S attach t1 } {} +do_test $tn.1.5 { S delete } {} +do_test $tn.1.6 { sqlite3session S db main } {S} +do_test $tn.1.7 { S attach t1 ; S attach t2 ; S attach t3 } {} +do_test $tn.1.8 { S attach t1 ; S attach t2 ; S attach t3 } {} +do_test $tn.1.9 { S delete } {} +do_test $tn.1.10 { sqlite3session S db main S attach t1 execsql { INSERT INTO t1 VALUES('ghi', 'jkl') } } {} -do_test 1.11 { S delete } {} -do_test 1.12 { - sqlite3session S db main - S attach t1 - execsql { INSERT INTO t1 VALUES('mno', 'pqr') } - execsql { UPDATE t1 SET x = 111 WHERE rowid = 1 } - execsql { DELETE FROM t1 WHERE rowid = 2 } -} {} -do_test 1.13 { - S changeset - S delete -} {} +do_test $tn.1.11 { S delete } {} +if {$tn==1} { + do_test $tn.1.12 { + sqlite3session S db main + S attach t1 + execsql { INSERT INTO t1 VALUES('mno', 'pqr') } + execsql { UPDATE t1 SET x = 111 WHERE rowid = 1 } + execsql { DELETE FROM t1 WHERE rowid = 2 } + } {} + do_test $tn.1.13 { + S changeset + S delete + } {} +} #------------------------------------------------------------------------- # Simple changeset tests. Also test the sqlite3changeset_invert() # function. # -do_test 2.1.1 { +do_test $tn.2.1.1 { execsql { DELETE FROM t1 } sqlite3session S db main S attach t1 @@ -67,36 +82,36 @@ do_test 2.1.1 { execsql { INSERT INTO t1 VALUES(2, 'Ayutthaya') } execsql { INSERT INTO t1 VALUES(3, 'Thonburi') } } {} -do_changeset_test 2.1.2 S { +do_changeset_test $tn.2.1.2 S { {INSERT t1 0 X. {} {i 1 t Sukhothai}} {INSERT t1 0 X. {} {i 2 t Ayutthaya}} {INSERT t1 0 X. {} {i 3 t Thonburi}} } -do_changeset_invert_test 2.1.3 S { +do_changeset_invert_test $tn.2.1.3 S { {DELETE t1 0 X. {i 1 t Sukhothai} {}} {DELETE t1 0 X. {i 2 t Ayutthaya} {}} {DELETE t1 0 X. {i 3 t Thonburi} {}} } -do_test 2.1.4 { S delete } {} +do_test $tn.2.1.4 { S delete } {} -do_test 2.2.1 { +do_test $tn.2.2.1 { sqlite3session S db main S attach t1 execsql { DELETE FROM t1 WHERE 1 } } {} -do_changeset_test 2.2.2 S { +do_changeset_test $tn.2.2.2 S { {DELETE t1 0 X. {i 1 t Sukhothai} {}} {DELETE t1 0 X. {i 2 t Ayutthaya} {}} {DELETE t1 0 X. {i 3 t Thonburi} {}} } -do_changeset_invert_test 2.2.3 S { +do_changeset_invert_test $tn.2.2.3 S { {INSERT t1 0 X. {} {i 1 t Sukhothai}} {INSERT t1 0 X. {} {i 2 t Ayutthaya}} {INSERT t1 0 X. {} {i 3 t Thonburi}} } -do_test 2.2.4 { S delete } {} +do_test $tn.2.2.4 { S delete } {} -do_test 2.3.1 { +do_test $tn.2.3.1 { execsql { DELETE FROM t1 } sqlite3session S db main execsql { INSERT INTO t1 VALUES(1, 'Sukhothai') } @@ -110,7 +125,7 @@ do_test 2.3.1 { } } {} -do_changeset_test 2.3.2 S { +do_changeset_test $tn.2.3.2 S { {INSERT t1 0 X. {} {i 10 t Sukhothai}} {DELETE t1 0 X. {i 1 t Sukhothai} {}} {UPDATE t1 0 X. {i 2 t Ayutthaya} {{} {} t Surin}} @@ -118,24 +133,24 @@ do_changeset_test 2.3.2 S { {INSERT t1 0 X. {} {i 20 t Thapae}} } -do_changeset_invert_test 2.3.3 S { +do_changeset_invert_test $tn.2.3.3 S { {DELETE t1 0 X. {i 10 t Sukhothai} {}} {INSERT t1 0 X. {} {i 1 t Sukhothai}} {UPDATE t1 0 X. {i 2 t Surin} {{} {} t Ayutthaya}} {INSERT t1 0 X. {} {i 3 t Thonburi}} {DELETE t1 0 X. {i 20 t Thapae} {}} } -do_test 2.3.4 { S delete } {} +do_test $tn.2.3.4 { S delete } {} -do_test 2.4.1 { +do_test $tn.2.4.1 { sqlite3session S db main S attach t1 execsql { INSERT INTO t1 VALUES(100, 'Bangkok') } execsql { DELETE FROM t1 WHERE x = 100 } } {} -do_changeset_test 2.4.2 S {} -do_changeset_invert_test 2.4.3 S {} -do_test 2.4.4 { S delete } {} +do_changeset_test $tn.2.4.2 S {} +do_changeset_invert_test $tn.2.4.3 S {} +do_test $tn.2.4.4 { S delete } {} #------------------------------------------------------------------------- # Test the application of simple changesets. These tests also test that @@ -189,16 +204,16 @@ proc do_db2_test {testname sql {result {}}} { # Test INSERT changesets. # -do_test 3.1.0 { - execsql { CREATE TABLE t1(a PRIMARY KEY, b NOT NULL) } db2 +do_test $tn.3.1.0 { + execsql { CREATE TABLE t1(a PRIMARY KEY, b NOT NULL) %WR% } db2 execsql { - CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t1(a PRIMARY KEY, b) %WR%; INSERT INTO t1 VALUES(1, 'one'); INSERT INTO t1 VALUES(2, 'two'); } db } {} -do_db2_test 3.1.1 "INSERT INTO t1 VALUES(6, 'VI')" -do_conflict_test 3.1.2 -tables t1 -sql { +do_db2_test $tn.3.1.1 "INSERT INTO t1 VALUES(6, 'VI')" +do_conflict_test $tn.3.1.2 -tables t1 -sql { INSERT INTO t1 VALUES(3, 'three'); INSERT INTO t1 VALUES(4, 'four'); INSERT INTO t1 VALUES(5, 'five'); @@ -210,34 +225,34 @@ do_conflict_test 3.1.2 -tables t1 -sql { {INSERT t1 CONSTRAINT {i 8 n {}}} } -do_db2_test 3.1.3 "SELECT * FROM t1" { - 6 VI 3 three 4 four 5 five 7 seven +do_db2_test $tn.3.1.3 "SELECT * FROM t1 ORDER BY a" { + 3 three 4 four 5 five 6 VI 7 seven } -do_execsql_test 3.1.4 "SELECT * FROM t1" { +do_execsql_test $tn.3.1.4 "SELECT * FROM t1" { 1 one 2 two 3 three 4 four 5 five 6 six 7 seven 8 {} } # Test DELETE changesets. # -do_execsql_test 3.2.1 { +do_execsql_test $tn.3.2.1 { PRAGMA foreign_keys = on; - CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b)%WR%; CREATE TABLE t3(c, d REFERENCES t2); INSERT INTO t2 VALUES(1, 'one'); INSERT INTO t2 VALUES(2, 'two'); INSERT INTO t2 VALUES(3, 'three'); INSERT INTO t2 VALUES(4, 'four'); } -do_db2_test 3.2.2 { +do_db2_test $tn.3.2.2 { PRAGMA foreign_keys = on; - CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b)%WR%; CREATE TABLE t3(c, d REFERENCES t2); INSERT INTO t2 VALUES(1, 'one'); INSERT INTO t2 VALUES(2, 'two'); INSERT INTO t2 VALUES(4, 'five'); INSERT INTO t3 VALUES('i', 1); } -do_conflict_test 3.2.3 -tables t2 -sql { +do_conflict_test $tn.3.2.3 -tables t2 -sql { DELETE FROM t2 WHERE a = 1; DELETE FROM t2 WHERE a = 2; DELETE FROM t2 WHERE a = 3; @@ -247,26 +262,26 @@ do_conflict_test 3.2.3 -tables t2 -sql { {DELETE t2 DATA {i 4 t four} {i 4 t five}} {FOREIGN_KEY 1} } -do_execsql_test 3.2.4 "SELECT * FROM t2" {} -do_db2_test 3.2.5 "SELECT * FROM t2" {4 five} +do_execsql_test $tn.3.2.4 "SELECT * FROM t2" {} +do_db2_test $tn.3.2.5 "SELECT * FROM t2" {4 five} # Test UPDATE changesets. # -do_execsql_test 3.3.1 { - CREATE TABLE t4(a, b, c, PRIMARY KEY(b, c)); +do_execsql_test $tn.3.3.1 { + CREATE TABLE t4(a, b, c, PRIMARY KEY(b, c))%WR%; INSERT INTO t4 VALUES(1, 2, 3); INSERT INTO t4 VALUES(4, 5, 6); INSERT INTO t4 VALUES(7, 8, 9); INSERT INTO t4 VALUES(10, 11, 12); } -do_db2_test 3.3.2 { - CREATE TABLE t4(a NOT NULL, b, c, PRIMARY KEY(b, c)); +do_db2_test $tn.3.3.2 { + CREATE TABLE t4(a NOT NULL, b, c, PRIMARY KEY(b, c))%WR%; INSERT INTO t4 VALUES(0, 2, 3); INSERT INTO t4 VALUES(4, 5, 7); INSERT INTO t4 VALUES(7, 8, 9); INSERT INTO t4 VALUES(10, 11, 12); } -do_conflict_test 3.3.3 -tables t4 -sql { +do_conflict_test $tn.3.3.3 -tables t4 -sql { UPDATE t4 SET a = -1 WHERE b = 2; UPDATE t4 SET a = -1 WHERE b = 5; UPDATE t4 SET a = NULL WHERE c = 9; @@ -276,8 +291,8 @@ do_conflict_test 3.3.3 -tables t4 -sql { {UPDATE t4 NOTFOUND {i 4 i 5 i 6} {i -1 {} {} {} {}}} {UPDATE t4 CONSTRAINT {i 7 i 8 i 9} {n {} {} {} {} {}}} } -do_db2_test 3.3.4 { SELECT * FROM t4 } {0 2 3 4 5 7 7 8 9 x 11 12} -do_execsql_test 3.3.5 { SELECT * FROM t4 } {-1 2 3 -1 5 6 {} 8 9 x 11 12} +do_db2_test $tn.3.3.4 { SELECT * FROM t4 } {0 2 3 4 5 7 7 8 9 x 11 12} +do_execsql_test $tn.3.3.5 { SELECT * FROM t4 } {-1 2 3 -1 5 6 {} 8 9 x 11 12} #------------------------------------------------------------------------- # This next block of tests verifies that values returned by the conflict @@ -297,16 +312,16 @@ proc xConflict {args} { return $::conflict_return } -foreach {tn conflict_return after} { +foreach {tn2 conflict_return after} { 1 OMIT {1 2 value1 4 5 7 10 x x} 2 REPLACE {1 2 value1 4 5 value2 10 8 9} } { test_reset - do_test 4.$tn.1 { + do_test $tn.4.$tn2.1 { foreach db {db db2} { execsql { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a)); + CREATE TABLE t1(a, b, c, PRIMARY KEY(a))%WR%; INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t1 VALUES(7, 8, 9); @@ -318,7 +333,7 @@ foreach {tn conflict_return after} { } db2 } {} - do_conflict_test 4.$tn.2 -tables t1 -sql { + do_conflict_test $tn.4.$tn2.2 -tables t1 -sql { UPDATE t1 SET c = 'value1' WHERE a = 1; -- no conflict UPDATE t1 SET c = 'value2' WHERE a = 4; -- DATA conflict UPDATE t1 SET a = 10 WHERE a = 7; -- CONFLICT conflict @@ -327,19 +342,19 @@ foreach {tn conflict_return after} { {UPDATE t1 DATA {i 4 {} {} i 6} {{} {} {} {} t value2} {i 4 i 5 i 7}} } - do_db2_test 4.$tn.3 "SELECT * FROM t1 ORDER BY a" $after + do_db2_test $tn.4.$tn2.3 "SELECT * FROM t1 ORDER BY a" $after } -foreach {tn conflict_return} { +foreach {tn2 conflict_return} { 1 OMIT 2 REPLACE } { test_reset - do_test 5.$tn.1 { + do_test $tn.5.$tn2.1 { # Create an identical schema in both databases. set schema { - CREATE TABLE "'foolish name'"(x, y, z, PRIMARY KEY(x, y)); + CREATE TABLE "'foolish name'"(x, y, z, PRIMARY KEY(x, y))%WR%; } execsql $schema db execsql $schema db2 @@ -354,7 +369,7 @@ foreach {tn conflict_return} { } {} - do_conflict_test 5.$tn.2 -tables {{'foolish name'}} -sql { + do_conflict_test $tn.5.$tn2.2 -tables {{'foolish name'}} -sql { INSERT INTO "'foolish name'" VALUES('one', 'two', 2); } -conflicts { {INSERT {'foolish name'} CONFLICT {t one t two i 2} {t one t two t i}} @@ -362,14 +377,14 @@ foreach {tn conflict_return} { set res(REPLACE) {one one ii one two 2 two two ii} set res(OMIT) {one one ii one two i two two ii} - do_db2_test 5.$tn.3 { + do_db2_test $tn.5.$tn2.3 { SELECT * FROM "'foolish name'" ORDER BY x, y } $res($conflict_return) - do_test 5.$tn.1 { + do_test $tn.5.$tn2.1 { set schema { - CREATE TABLE d1("z""z" PRIMARY KEY, y); + CREATE TABLE d1("z""z" PRIMARY KEY, y)%WR%; INSERT INTO d1 VALUES(1, 'one'); INSERT INTO d1 VALUES(2, 'two'); } @@ -382,7 +397,7 @@ foreach {tn conflict_return} { } {} - do_conflict_test 5.$tn.2 -tables d1 -sql { + do_conflict_test $tn.5.$tn2.2 -tables d1 -sql { DELETE FROM d1 WHERE "z""z" = 2; } -conflicts { {DELETE d1 DATA {i 2 t two} {i 2 t TWO}} @@ -390,7 +405,7 @@ foreach {tn conflict_return} { set res(REPLACE) {1 one} set res(OMIT) {1 one 2 TWO} - do_db2_test 5.$tn.3 "SELECT * FROM d1" $res($conflict_return) + do_db2_test $tn.5.$tn2.3 "SELECT * FROM d1" $res($conflict_return) } #------------------------------------------------------------------------- @@ -398,10 +413,10 @@ foreach {tn conflict_return} { # test_reset set schema { - CREATE TABLE t1(a COLLATE nocase PRIMARY KEY, b); - CREATE TABLE t2(a, b PRIMARY KEY); + CREATE TABLE t1(a COLLATE nocase PRIMARY KEY, b)%WR%; + CREATE TABLE t2(a, b PRIMARY KEY)%WR%; } -do_test 6.0 { +do_test $tn.6.0 { execsql $schema db execsql $schema db2 execsql { @@ -411,7 +426,7 @@ do_test 6.0 { } {} set conflict_return "" -do_conflict_test 6.1 -tables {t1 t2} -sql { +do_conflict_test $tn.6.1 -tables {t1 t2} -sql { INSERT INTO t1 VALUES('1', '2'); INSERT INTO t1 VALUES('A', 'B'); INSERT INTO t2 VALUES('A', 'B'); @@ -419,8 +434,8 @@ do_conflict_test 6.1 -tables {t1 t2} -sql { {INSERT t1 CONFLICT {t A t B} {t a t b}} } -do_db2_test 6.2 "SELECT * FROM t1" {a b 1 2} -do_db2_test 6.3 "SELECT * FROM t2" {a b A B} +do_db2_test $tn.6.2 "SELECT * FROM t1 ORDER BY a" {1 2 a b} +do_db2_test $tn.6.3 "SELECT * FROM t2 ORDER BY a" {A B a b} #------------------------------------------------------------------------- # Test that session objects are not confused by changes to table in @@ -429,10 +444,10 @@ do_db2_test 6.3 "SELECT * FROM t2" {a b A B} catch { db2 close } drop_all_tables forcedelete test.db2 -do_iterator_test 7.1 * { +do_iterator_test $tn.7.1 * { ATTACH 'test.db2' AS aux; - CREATE TABLE main.t1(x PRIMARY KEY, y); - CREATE TABLE aux.t1(x PRIMARY KEY, y); + CREATE TABLE main.t1(x PRIMARY KEY, y)%WR%; + CREATE TABLE aux.t1(x PRIMARY KEY, y)%WR%; INSERT INTO main.t1 VALUES('one', 1); INSERT INTO main.t1 VALUES('two', 2); @@ -446,10 +461,10 @@ do_iterator_test 7.1 * { #------------------------------------------------------------------------- # Test the sqlite3session_isempty() function. # -do_test 8.1 { +do_test $tn.8.1 { execsql { - CREATE TABLE t5(x PRIMARY KEY, y); - CREATE TABLE t6(x PRIMARY KEY, y); + CREATE TABLE t5(x PRIMARY KEY, y)%WR%; + CREATE TABLE t6(x PRIMARY KEY, y)%WR%; INSERT INTO t5 VALUES('a', 'b'); INSERT INTO t6 VALUES('a', 'b'); } @@ -458,20 +473,20 @@ do_test 8.1 { S isempty } {1} -do_test 8.2 { +do_test $tn.8.2 { execsql { DELETE FROM t5 } S isempty } {0} -do_test 8.3 { +do_test $tn.8.3 { S delete sqlite3session S db main S attach t5 execsql { DELETE FROM t5 } S isempty } {1} -do_test 8.4 { S delete } {} +do_test $tn.8.4 { S delete } {} -do_test 8.5 { +do_test $tn.8.5 { sqlite3session S db main S attach t5 S attach t6 @@ -479,7 +494,7 @@ do_test 8.5 { S isempty } {0} -do_test 8.6 { +do_test $tn.8.6 { S delete sqlite3session S db main S attach t5 @@ -487,20 +502,20 @@ do_test 8.6 { execsql { INSERT INTO t6 VALUES(1, 2) } S isempty } {0} -do_test 8.7 { S delete } {} +do_test $tn.8.7 { S delete } {} #------------------------------------------------------------------------- # -do_execsql_test 9.1 { - CREATE TABLE t7(a, b, c, d, e PRIMARY KEY, f, g); +do_execsql_test $tn.9.1 { + CREATE TABLE t7(a, b, c, d, e PRIMARY KEY, f, g)%WR%; INSERT INTO t7 VALUES(1, 1, 1, 1, 1, 1, 1); } -do_test 9.2 { +do_test $tn.9.2 { sqlite3session S db main S attach * execsql { UPDATE t7 SET b=2, d=2 } } {} -do_changeset_test 9.2 S {{UPDATE t7 0 ....X.. {{} {} i 1 {} {} i 1 i 1 {} {} {} {}} {{} {} i 2 {} {} i 2 {} {} {} {} {} {}}}} +do_changeset_test $tn.9.2 S {{UPDATE t7 0 ....X.. {{} {} i 1 {} {} i 1 i 1 {} {} {} {}} {{} {} i 2 {} {} i 2 {} {} {} {} {} {}}}} S delete catch { db2 close } @@ -509,9 +524,9 @@ catch { db2 close } # reset_db set tblname [string repeat tblname123 100] -do_test 10.1.1 { +do_test $tn.10.1.1 { execsql " - CREATE TABLE $tblname (a PRIMARY KEY, b); + CREATE TABLE $tblname (a PRIMARY KEY, b)%WR%; INSERT INTO $tblname VALUES('xyz', 'def'); " sqlite3session S db main @@ -522,18 +537,18 @@ do_test 10.1.1 { " } {} breakpoint -do_changeset_test 10.1.2 S " +do_changeset_test $tn.10.1.2 S " {INSERT $tblname 0 X. {} {t uvw t abc}} {DELETE $tblname 0 X. {t xyz t def} {}} " -do_test 10.1.4 { S delete } {} +do_test $tn.10.1.4 { S delete } {} #--------------------------------------------------------------- reset_db -do_execsql_test 11.1 { +do_execsql_test $tn.11.1 { CREATE TABLE t1(a, b); } -do_test 11.2 { +do_test $tn.11.2 { sqlite3session S db main S attach t1 execsql { @@ -550,9 +565,9 @@ S delete # reset_db set tblname [string repeat tblname123 100] -do_test 10.1.1 { +do_test $tn.10.1.1 { execsql " - CREATE TABLE $tblname (a PRIMARY KEY, b); + CREATE TABLE $tblname (a PRIMARY KEY, b)%WR%; INSERT INTO $tblname VALUES('xyz', 'def'); " sqlite3session S db main @@ -563,35 +578,35 @@ do_test 10.1.1 { " } {} breakpoint -do_changeset_test 10.1.2 S " +do_changeset_test $tn.10.1.2 S " {INSERT $tblname 0 X. {} {t uvw t abc}} {DELETE $tblname 0 X. {t xyz t def} {}} " -do_test 10.1.4 { S delete } {} +do_test $tn.10.1.4 { S delete } {} #------------------------------------------------------------------------- # Test the effect of updating a column from 0.0 to 0.0. # reset_db -do_execsql_test 11.1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b REAL); +do_execsql_test $tn.11.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b REAL)%WR%; INSERT INTO t1 VALUES(1, 0.0); } -do_iterator_test 11.2 * { +do_iterator_test $tn.11.2 * { UPDATE t1 SET b = 0.0; } { } reset_db -do_execsql_test 12.1 { - CREATE TABLE t1(r INTEGER PRIMARY KEY, a, b); +do_execsql_test $tn.12.1 { + CREATE TABLE t1(r INTEGER PRIMARY KEY, a, b)%WR%; CREATE INDEX i1 ON t1(a); INSERT INTO t1 VALUES(1, 1, 1); INSERT INTO t1 VALUES(2, 1, 2); INSERT INTO t1 VALUES(3, 1, 3); } -do_iterator_test 12.2 * { +do_iterator_test $tn.12.2 * { UPDATE t1 SET b='one' WHERE a=1; } { {UPDATE t1 0 X.. {i 1 {} {} i 1} {{} {} {} {} t one}} @@ -599,6 +614,8 @@ do_iterator_test 12.2 * { {UPDATE t1 0 X.. {i 3 {} {} i 3} {{} {} {} {} t one}} } +}] +} finish_test diff --git a/ext/session/sessionwor.test b/ext/session/sessionwor.test new file mode 100644 index 0000000000..d906a92b62 --- /dev/null +++ b/ext/session/sessionwor.test @@ -0,0 +1,57 @@ +# 2017 Jan 31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. Specifically, +# testing support for WITHOUT ROWID tables. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionwor + +proc test_reset {} { + catch { db close } + catch { db2 close } + forcedelete test.db test.db2 + sqlite3 db test.db + sqlite3 db2 test.db2 +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b) WITHOUT ROWID; +} + +do_iterator_test 1.1 t1 { + INSERT INTO t1 VALUES('one', 'two'); +} { + {INSERT t1 0 X. {} {t one t two}} +} + +do_iterator_test 1.2 t1 { + UPDATE t1 SET b='three' +} { + {UPDATE t1 0 X. {t one t two} {{} {} t three}} +} + +do_iterator_test 1.3 t1 { + DELETE FROM t1; +} { + {DELETE t1 0 X. {t one t three} {}} +} + +finish_test + diff --git a/manifest b/manifest index 229623bcc5..c93ba97d64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbuilding\swith\sSQLITE_OMIT_FOREIGN_KEY\sdefined. -D 2017-01-30T11:38:19.468 +C Experimental\schange\sto\sinvoke\sthe\spreupdate\shook\swhen\sWITHOUT\sROWID\stables\sare\nwritten. +D 2017-01-30T19:44:53.682 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -288,7 +288,7 @@ F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a -F ext/session/session1.test 482e9b861169d0b7e283e1563a28c0fa32779fa5 +F ext/session/session1.test c8a50e0e8581dc1a00e832aa59bb61f180404d44 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 @@ -306,6 +306,7 @@ F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 +F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc F ext/session/sqlite3session.c c61a43396368ec00dc127f7bc647e9bd6a4ee5fb F ext/session/sqlite3session.h 9345166bd8f80562145586cf817f707de5ecada2 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 @@ -353,7 +354,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 8183e1778ec42d26732bec8c98aeffa8af13038f +F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -458,11 +459,11 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c c27cc34be1d9169c1c191238025781684bdcd4ec +F src/vdbe.c 57f41c8c74ceb757abbd4cda72c44fc40d57f425 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c -F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e -F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 -F src/vdbeaux.c 6847b02aa2db536ed15d90f1fdc2923afef93c5b +F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f +F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 +F src/vdbeaux.c b9a36e530e6525ca9d9a685bc7b1d01fa77b5cf8 F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -834,6 +835,7 @@ F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 +F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -1548,7 +1550,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4fe879d4b5da6ae0688a7a99004683a234966597 -R c0ed5c24fdff676195cd77401e89700b +P e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab +R f7cc20c31f0badea537c676e1225b282 +T *branch * preupdate-without-rowid +T *sym-preupdate-without-rowid * +T -sym-trunk * U dan -Z 6d1b138d17ccacee0c622f2ed331ca89 +Z 812bce2fe1496eddb1cec5705157fb44 diff --git a/manifest.uuid b/manifest.uuid index eec33fc0a4..67552c1672 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab \ No newline at end of file +856f8604c59c8fdd9bfb7d86fc0e212f091ab49a \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 5370ef77ed..894bfc2cc1 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1730,16 +1730,23 @@ void sqlite3CompleteInsertion( sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); } - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], - aRegIdx[i]+1, - pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); - pik_flags = 0; - if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( update_flags==0 ){ + sqlite3VdbeAddOp4(v, OP_InsertInt, + iIdxCur+i, aRegIdx[i], 0, (char*)pTab, P4_TABLE + ); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + } +#endif } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; diff --git a/src/vdbe.c b/src/vdbe.c index ded4462398..5462e541c5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4384,7 +4384,7 @@ case OP_InsertInt: { assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); - assert( pC->isTable ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable ); assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); @@ -4400,11 +4400,10 @@ case OP_InsertInt: { } if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ - assert( pC->isTable ); assert( pC->iDb>=0 ); zDb = db->aDb[pC->iDb].zDbSName; pTab = pOp->p4.pTab; - assert( HasRowid(pTab) ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) ); op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); }else{ pTab = 0; /* Not needed. Silence a comiler warning. */ @@ -4419,6 +4418,7 @@ case OP_InsertInt: { ){ sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2); } + if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; @@ -4531,8 +4531,11 @@ case OP_Delete: { #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update-hook if required. */ - if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){ - assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) ); + if( db->xPreUpdateCallback && pOp->p4.pTab ){ + assert( !(opflags & OPFLAG_ISUPDATE) + || HasRowid(pTab)==0 + || (aMem[pOp->p3].flags & MEM_Int) + ); sqlite3VdbePreUpdateHook(p, pC, (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, zDb, pTab, pC->movetoTarget, diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 3e6bb06889..989cdfd346 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -434,6 +434,7 @@ struct PreUpdate { i64 iKey2; /* Second key value passed to hook */ Mem *aNew; /* Array of new.* values */ Table *pTab; /* Schema object being upated */ + Index *pPk; /* PK index if pTab is WITHOUT ROWID */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 7ecdac87c3..6eb97f1d1d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1669,6 +1669,9 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ rc = SQLITE_MISUSE_BKPT; goto preupdate_old_out; } + if( p->pPk ){ + iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; @@ -1754,6 +1757,9 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; } + if( p->pPk && p->op!=SQLITE_UPDATE ){ + iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_new_out; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 23c31d4029..68f6a5acc8 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -4620,10 +4620,15 @@ void sqlite3VdbePreUpdateHook( assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); - if( op==SQLITE_UPDATE ){ - iKey2 = v->aMem[iReg].u.i; + if( HasRowid(pTab)==0 ){ + iKey1 = iKey2 = 0; + preupdate.pPk = sqlite3PrimaryKeyIndex(pTab); }else{ - iKey2 = iKey1; + if( op==SQLITE_UPDATE ){ + iKey2 = v->aMem[iReg].u.i; + }else{ + iKey2 = iKey1; + } } assert( pCsr->nField==pTab->nCol diff --git a/test/hook2.test b/test/hook2.test new file mode 100644 index 0000000000..9ae1103a7d --- /dev/null +++ b/test/hook2.test @@ -0,0 +1,218 @@ +# 2017 Jan 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# The tests in this file focus on the pre-update hook. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix hook2 + +ifcapable !preupdate { + finish_test + return +} + +#------------------------------------------------------------------------- +proc do_preupdate_test {tn sql x} { + set X [list] + foreach elem $x {lappend X $elem} + uplevel do_test $tn [list " + set ::preupdate \[list\] + execsql { $sql } + set ::preupdate + "] [list $X] +} + +proc preupdate_hook {args} { + set type [lindex $args 0] + eval lappend ::preupdate $args + if {$type != "INSERT"} { + for {set i 0} {$i < [db preupdate count]} {incr i} { + lappend ::preupdate [db preupdate old $i] + } + } + if {$type != "DELETE"} { + for {set i 0} {$i < [db preupdate count]} {incr i} { + set rc [catch { db preupdate new $i } v] + lappend ::preupdate $v + } + } +} + +#------------------------------------------------------------------------- +# Simple tests - INSERT, UPDATE and DELETE on a WITHOUT ROWID table. +# +db preupdate hook preupdate_hook +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b) WITHOUT ROWID; +} +do_preupdate_test 1.1 { + INSERT INTO t1 VALUES('one', 1); +} { + INSERT main t1 0 0 one 1 +} +do_preupdate_test 1.2 { + UPDATE t1 SET b=2 WHERE a='one'; +} { + UPDATE main t1 0 0 one 1 one 2 +} +do_preupdate_test 1.3 { + DELETE FROM t1 WHERE a='one'; +} { + DELETE main t1 0 0 one 2 +} + +#------------------------------------------------------------------------- +# Some more complex tests for the pre-update callback on WITHOUT ROWID +# tables. +# +# 2.1.1 - INSERT statement. +# 2.1.2 - INSERT INTO ... SELECT statement. +# 2.1.3 - REPLACE INTO ... (PK conflict) +# 2.1.4 - REPLACE INTO ... (other index conflicts) +# 2.1.5 - REPLACE INTO ... (both PK and other index conflicts) +# +# 2.2.1 - DELETE statement. +# 2.2.2 - DELETE statement that uses the truncate optimization. +# +# 2.3.1 - UPDATE statement. +# 2.3.2 - UPDATE statement that modifies the PK. +# 2.3.3 - UPDATE OR REPLACE ... (PK conflict). +# 2.3.4 - UPDATE OR REPLACE ... (other index conflicts) +# 2.3.4 - UPDATE OR REPLACE ... (both PK and other index conflicts) +# +do_execsql_test 2.0 { + CREATE TABLE t2(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + CREATE UNIQUE INDEX t2a ON t2(a); +} + +do_preupdate_test 2.1.1 { + INSERT INTO t2(b, c) VALUES(1, 1); +} { + INSERT main t2 0 0 4 1 1 +} + +do_execsql_test 2.1.2.0 { + CREATE TABLE d1(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + CREATE UNIQUE INDEX d1a ON d1(a); + INSERT INTO d1 VALUES(1, 2, 3); + INSERT INTO d1 VALUES(11, 12, 13); +} +do_preupdate_test 2.1.2.1 { + INSERT INTO t2 SELECT * FROM d1; +} { + INSERT main t2 0 0 1 2 3 + INSERT main t2 0 0 11 12 13 +} +do_preupdate_test 2.1.2.2 { + INSERT INTO t2 SELECT a+20, b+20, c+20 FROM d1; +} { + INSERT main t2 0 0 21 22 23 + INSERT main t2 0 0 31 32 33 +} +do_execsql_test 2.1.2.3 { + SELECT * FROM t2 ORDER BY b, c; +} { + 4 1 1 + 1 2 3 + 11 12 13 + 21 22 23 + 31 32 33 +} +do_preupdate_test 2.1.3 { + REPLACE INTO t2 VALUES(45, 22, 23); +} { + DELETE main t2 0 0 21 22 23 + INSERT main t2 0 0 45 22 23 +} +do_preupdate_test 2.1.4 { + REPLACE INTO t2 VALUES(11, 100, 100); +} { + DELETE main t2 0 0 11 12 13 + INSERT main t2 0 0 11 100 100 +} +do_preupdate_test 2.1.5 { + REPLACE INTO t2(c, b) VALUES(33, 32) +} { + DELETE main t2 0 0 4 1 1 + DELETE main t2 0 0 31 32 33 + INSERT main t2 0 0 4 32 33 +} + +do_execsql_test 2.2.0 { + SELECT * FROM t2 ORDER BY b,c; +} { + 1 2 3 + 45 22 23 + 4 32 33 + 11 100 100 +} +do_preupdate_test 2.2.1 { + DELETE FROM t2 WHERE b=22; +} { + DELETE main t2 0 0 45 22 23 +} +do_preupdate_test 2.2.2 { + DELETE FROM t2; +} { + DELETE main t2 0 0 1 2 3 + DELETE main t2 0 0 4 32 33 + DELETE main t2 0 0 11 100 100 +} + +do_execsql_test 2.3.0 { + CREATE TABLE t3(x, y PRIMARY KEY, z UNIQUE) WITHOUT ROWID; + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + + INSERT INTO t3 VALUES(1, 1, 1); + INSERT INTO t3 VALUES(2, 2, 2); + INSERT INTO t3 VALUES(3, 3, 3); +} + +do_preupdate_test 2.3.1 { + UPDATE t3 SET x=4 WHERE y IN ('b', 'e', 'x'); +} { + UPDATE main t3 0 0 a b c 4 b c + UPDATE main t3 0 0 d e f 4 e f +} + +do_preupdate_test 2.3.2 { + UPDATE t3 SET y=y||y WHERE z IN('c', 'f'); +} { + UPDATE main t3 0 0 4 b c 4 bb c + UPDATE main t3 0 0 4 e f 4 ee f +} + +do_preupdate_test 2.3.3 { + UPDATE OR REPLACE t3 SET y='bb' WHERE z='f' +} { + DELETE main t3 0 0 4 bb c + UPDATE main t3 0 0 4 ee f 4 bb f +} + +do_preupdate_test 2.3.4 { + UPDATE OR REPLACE t3 SET z=2 WHERE y=1; +} { + DELETE main t3 0 0 2 2 2 + UPDATE main t3 0 0 1 1 1 1 1 2 +} + +do_preupdate_test 2.3.5 { + UPDATE OR REPLACE t3 SET z=2, y='bb' WHERE y=3; +} { + DELETE main t3 0 0 1 1 2 + DELETE main t3 0 0 4 bb f + UPDATE main t3 0 0 3 3 3 3 bb 2 +} + + +finish_test From 18814dfb7eb1811c7e993860a52246bee3743b94 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 03:52:34 +0000 Subject: [PATCH 084/292] Performance optimization in sqlite3ExprAssignVarNumber(). FossilOrigin-Name: 5987ca1ff94ed3c1666f783bb15b16158aa7e1db --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 8 +++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 229623bcc5..96a31247b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbuilding\swith\sSQLITE_OMIT_FOREIGN_KEY\sdefined. -D 2017-01-30T11:38:19.468 +C Performance\soptimization\sin\ssqlite3ExprAssignVarNumber(). +D 2017-01-31T03:52:34.839 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -344,7 +344,7 @@ F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c f06f41e5e5daca10fb090e70a2502dcc0dbc992b +F src/expr.c 27ac552489a34fc85202084dda32d0a2a0664bfb F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4fe879d4b5da6ae0688a7a99004683a234966597 -R c0ed5c24fdff676195cd77401e89700b -U dan -Z 6d1b138d17ccacee0c622f2ed331ca89 +P e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab +R e1972f457267c7c1c09dde68002ccdbd +U drh +Z 4723986ca78be7a36581ca020c177062 diff --git a/manifest.uuid b/manifest.uuid index eec33fc0a4..5663814ec3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab \ No newline at end of file +5987ca1ff94ed3c1666f783bb15b16158aa7e1db \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 154d078ff7..9bf8936765 100644 --- a/src/expr.c +++ b/src/expr.c @@ -967,7 +967,13 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } x = (ynVar)i; testcase( i==0 ); testcase( i==1 ); From 8e74e7ba4c54ca46412bbfe3689d589a4b0d625a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 12:41:48 +0000 Subject: [PATCH 085/292] Further minor enhancement and size reduction in sqlite3ExprAssignVarNumber(). FossilOrigin-Name: eacfdcf25796ea29b5e63499c3d7397498305ad9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 96a31247b7..5c0df4b7d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\ssqlite3ExprAssignVarNumber(). -D 2017-01-31T03:52:34.839 +C Further\sminor\senhancement\sand\ssize\sreduction\sin\s\nsqlite3ExprAssignVarNumber(). +D 2017-01-31T12:41:48.800 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -344,7 +344,7 @@ F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c 27ac552489a34fc85202084dda32d0a2a0664bfb +F src/expr.c d7ef6fdb70bc18259d7d36b22aa0bc4845c5c5b9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e93d2c49a44af994ff10cc9cc7eafacd5a4f73ab -R e1972f457267c7c1c09dde68002ccdbd +P 5987ca1ff94ed3c1666f783bb15b16158aa7e1db +R 877cd166cfa1886b85d3c173dceec467 U drh -Z 4723986ca78be7a36581ca020c177062 +Z 12e7b02d2001dbb9bba0a7e780630702 diff --git a/manifest.uuid b/manifest.uuid index 5663814ec3..e94ba71f96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5987ca1ff94ed3c1666f783bb15b16158aa7e1db \ No newline at end of file +eacfdcf25796ea29b5e63499c3d7397498305ad9 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9bf8936765..18035bd421 100644 --- a/src/expr.c +++ b/src/expr.c @@ -974,7 +974,6 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ }else{ bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); } - x = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -984,6 +983,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); return; } + x = (ynVar)i; if( x>pParse->nVar ){ pParse->nVar = (int)x; doAdd = 1; From 1379521f4a6ef1ee68abb457a56f81292af69cb0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 15:27:04 +0000 Subject: [PATCH 086/292] Fix a typo in a comment. FossilOrigin-Name: bd22bf9cbe028e9811ca3afaadafd90312cb0fc9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5c0df4b7d4..8e4b6f544a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sminor\senhancement\sand\ssize\sreduction\sin\s\nsqlite3ExprAssignVarNumber(). -D 2017-01-31T12:41:48.800 +C Fix\sa\stypo\sin\sa\scomment. +D 2017-01-31T15:27:04.928 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -458,7 +458,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c c27cc34be1d9169c1c191238025781684bdcd4ec +F src/vdbe.c a5bcdb61862cd995126ec4993121e04fe409c6e0 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1548,7 +1548,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5987ca1ff94ed3c1666f783bb15b16158aa7e1db -R 877cd166cfa1886b85d3c173dceec467 +P eacfdcf25796ea29b5e63499c3d7397498305ad9 +R 08311637f9a8170c29c38d1bfabeb653 U drh -Z 12e7b02d2001dbb9bba0a7e780630702 +Z 855f1082721037fd6fefeb16ca2ac959 diff --git a/manifest.uuid b/manifest.uuid index e94ba71f96..61e2f28112 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eacfdcf25796ea29b5e63499c3d7397498305ad9 \ No newline at end of file +bd22bf9cbe028e9811ca3afaadafd90312cb0fc9 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index ded4462398..0e3a6f5baa 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4407,7 +4407,7 @@ case OP_InsertInt: { assert( HasRowid(pTab) ); op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); }else{ - pTab = 0; /* Not needed. Silence a comiler warning. */ + pTab = 0; /* Not needed. Silence a compiler warning. */ zDb = 0; /* Not needed. Silence a compiler warning. */ } From 9dc7ad1d7e1286fe0d3d30117adae154c6abf6de Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 15:29:05 +0000 Subject: [PATCH 087/292] Add the "stat" command to kvtest.c. Also add the --variance option to the "init" command. Add the tool/kvtest-speed.sh script used for doing performance testing on key/value access patterns. FossilOrigin-Name: b63deed600b1a457a6960ebad5645f4de9c56e5d --- manifest | 13 +++---- manifest.uuid | 2 +- test/kvtest.c | 80 ++++++++++++++++++++++++++++++++++++++++++-- tool/kvtest-speed.sh | 36 ++++++++++++++++++++ 4 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 tool/kvtest-speed.sh diff --git a/manifest b/manifest index 8e4b6f544a..e0848cf122 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment. -D 2017-01-31T15:27:04.928 +C Add\sthe\s"stat"\scommand\sto\skvtest.c.\s\sAlso\sadd\sthe\s--variance\soption\sto\sthe\n"init"\scommand.\s\sAdd\sthe\stool/kvtest-speed.sh\sscript\sused\sfor\sdoing\s\nperformance\stesting\son\skey/value\saccess\spatterns. +D 2017-01-31T15:29:05.242 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -899,7 +899,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c dc6e5e9066fa5e19f7368c7dfbe8c6b9642a706f +F test/kvtest.c 7a3c38ee56b9cc45dc5a5edc50fd9bc9425659a9 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1475,6 +1475,7 @@ F tool/fuzzershell.c dbf6c26eef936ec78cb0707570de3a4308b2507e F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +F tool/kvtest-speed.sh d6c7c2b5787f44c3be2ed01c8463223032ee598b F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7 F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 @@ -1548,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eacfdcf25796ea29b5e63499c3d7397498305ad9 -R 08311637f9a8170c29c38d1bfabeb653 +P bd22bf9cbe028e9811ca3afaadafd90312cb0fc9 +R 561f5ef3a58c98c17f69cad51ae91935 U drh -Z 855f1082721037fd6fefeb16ca2ac959 +Z 35fd92fc5491c4b603aa832f4a4ebf7a diff --git a/manifest.uuid b/manifest.uuid index 61e2f28112..69a91341ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd22bf9cbe028e9811ca3afaadafd90312cb0fc9 \ No newline at end of file +b63deed600b1a457a6960ebad5645f4de9c56e5d \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 765a7ee968..2a108aa470 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -67,13 +67,19 @@ static const char zHelp[] = "\n" " Generate a new test database file named DBFILE containing N\n" " BLOBs each of size M bytes. The page size of the new database\n" -" file will be X\n" +" file will be X. Additional options:\n" +"\n" +" --variance V Randomly vary M by plus or minus V\n" "\n" " kvtest export DBFILE DIRECTORY\n" "\n" " Export all the blobs in the kv table of DBFILE into separate\n" " files in DIRECTORY.\n" "\n" +" kvtest stat DBFILE\n" +"\n" +" Display summary information about DBFILE\n" +"\n" " kvtest run DBFILE [options]\n" "\n" " Run a performance test. DBFILE can be either the name of a\n" @@ -251,6 +257,7 @@ static int initMain(int argc, char **argv){ int i, rc; int nCount = 1000; int sz = 10000; + int iVariance = 0; int pgsz = 4096; sqlite3 *db; char *zSql; @@ -275,6 +282,11 @@ static int initMain(int argc, char **argv){ if( sz<1 ) fatalError("the --size must be positive"); continue; } + if( strcmp(z, "-variance")==0 ){ + if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); + iVariance = integerValue(argv[++i]); + continue; + } if( strcmp(z, "-pagesize")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); pgsz = integerValue(argv[++i]); @@ -296,9 +308,9 @@ static int initMain(int argc, char **argv){ "BEGIN;\n" "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" - " INSERT INTO kv(k,v) SELECT x, randomblob(%d) FROM c;\n" + " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n" "COMMIT;\n", - pgsz, nCount, sz + pgsz, nCount, sz, iVariance ); rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); if( rc ) fatalError("database create failed: %s", zErrMsg); @@ -307,6 +319,65 @@ static int initMain(int argc, char **argv){ return 0; } +/* +** Analyze an existing database file. Report its content. +*/ +static int statMain(int argc, char **argv){ + char *zDb; + int i, rc; + sqlite3 *db; + char *zSql; + sqlite3_stmt *pStmt; + + assert( strcmp(argv[1],"stat")==0 ); + assert( argc>=3 ); + zDb = argv[2]; + for(i=3; i&1 | tee summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME +cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-sql-$NAME.txt + +# Second run using the sqlite3_blob object +rm cachegrind.out.[1-9][0-9]* +valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME +cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt + +# Diff the sqlite3_blob API analysis for non-trunk runs. +if test "$NAME" != "trunk"; then + fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & +fi From 78a9e92ecf15afb4a3550c8bb8f998928fae3314 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 16:34:51 +0000 Subject: [PATCH 088/292] Remove an unnecessary initialization of the pOp variable in sqlite3VdbeExec(). FossilOrigin-Name: 02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e0848cf122..6b12ff4cc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"stat"\scommand\sto\skvtest.c.\s\sAlso\sadd\sthe\s--variance\soption\sto\sthe\n"init"\scommand.\s\sAdd\sthe\stool/kvtest-speed.sh\sscript\sused\sfor\sdoing\s\nperformance\stesting\son\skey/value\saccess\spatterns. -D 2017-01-31T15:29:05.242 +C Remove\san\sunnecessary\sinitialization\sof\sthe\spOp\svariable\sin\ssqlite3VdbeExec(). +D 2017-01-31T16:34:51.236 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -458,7 +458,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c a5bcdb61862cd995126ec4993121e04fe409c6e0 +F src/vdbe.c 7c53c4b66e5fdf4a59272a7af3826d6a36eb72ae F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bd22bf9cbe028e9811ca3afaadafd90312cb0fc9 -R 561f5ef3a58c98c17f69cad51ae91935 +P b63deed600b1a457a6960ebad5645f4de9c56e5d +R 30ecebc065226e07a2ed70b2eff6d7d3 U drh -Z 35fd92fc5491c4b603aa832f4a4ebf7a +Z 105450e93cb94372d4615eab06f395ed diff --git a/manifest.uuid b/manifest.uuid index 69a91341ca..7c7400a4e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b63deed600b1a457a6960ebad5645f4de9c56e5d \ No newline at end of file +02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 0e3a6f5baa..c6918c5f2a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -562,7 +562,7 @@ int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ Op *aOp = p->aOp; /* Copy of p->aOp */ - Op *pOp = aOp; /* Current operation */ + Op *pOp; /* Current operation */ #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) Op *pOrigOp; /* Value of pOp at the top of the loop */ #endif From d59bcbdee7241f103b0ec9ed456d0ce3637ac2b0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 16:43:36 +0000 Subject: [PATCH 089/292] Remove another unnecessary local variable initialization from sqlite3VdbeExec() FossilOrigin-Name: 2361b03b61311aab9b9ec9de040bbb73be31be0d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6b12ff4cc5..763bbd12a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sinitialization\sof\sthe\spOp\svariable\sin\ssqlite3VdbeExec(). -D 2017-01-31T16:34:51.236 +C Remove\sanother\sunnecessary\slocal\svariable\sinitialization\sfrom\ssqlite3VdbeExec() +D 2017-01-31T16:43:36.486 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -458,7 +458,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 7c53c4b66e5fdf4a59272a7af3826d6a36eb72ae +F src/vdbe.c c0e9bd3fc0f1909e76930e88860ec66edc201d62 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b63deed600b1a457a6960ebad5645f4de9c56e5d -R 30ecebc065226e07a2ed70b2eff6d7d3 +P 02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 +R f0488c71d9bf7cc95008ff5cdb6232cc U drh -Z 105450e93cb94372d4615eab06f395ed +Z 567d0d39f344217652f484cd397a1b36 diff --git a/manifest.uuid b/manifest.uuid index 7c7400a4e6..1b110b97c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 \ No newline at end of file +2361b03b61311aab9b9ec9de040bbb73be31be0d \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c6918c5f2a..b133c7ec85 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -597,7 +597,7 @@ int sqlite3VdbeExec( } assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); - p->rc = SQLITE_OK; +// p->rc = SQLITE_OK; p->iCurrentTime = 0; assert( p->explain==0 ); p->pResultSet = 0; From 0caad09585bd83f287fcb31c968d656c4dbe9fda Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 16:49:01 +0000 Subject: [PATCH 090/292] Remove a C99-style comment. Fixes to the kvtest-speed.sh script. FossilOrigin-Name: 91eb6b628e278d20eccc647293e5b30765163e12 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 - tool/kvtest-speed.sh | 5 ++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 763bbd12a7..3c10f176f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sanother\sunnecessary\slocal\svariable\sinitialization\sfrom\ssqlite3VdbeExec() -D 2017-01-31T16:43:36.486 +C Remove\sa\sC99-style\scomment.\s\sFixes\sto\sthe\skvtest-speed.sh\sscript. +D 2017-01-31T16:49:01.913 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -458,7 +458,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c c0e9bd3fc0f1909e76930e88860ec66edc201d62 +F src/vdbe.c d3765f4d8f667b8d10232b2a1d0498497f3ac09f F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 @@ -1475,7 +1475,7 @@ F tool/fuzzershell.c dbf6c26eef936ec78cb0707570de3a4308b2507e F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/kvtest-speed.sh d6c7c2b5787f44c3be2ed01c8463223032ee598b +F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7 F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 -R f0488c71d9bf7cc95008ff5cdb6232cc +P 2361b03b61311aab9b9ec9de040bbb73be31be0d +R 085c25b2bd73ed43799df664becffebf U drh -Z 567d0d39f344217652f484cd397a1b36 +Z 9d1629688efb4c919cbb0184aee51efa diff --git a/manifest.uuid b/manifest.uuid index 1b110b97c9..80341cb011 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2361b03b61311aab9b9ec9de040bbb73be31be0d \ No newline at end of file +91eb6b628e278d20eccc647293e5b30765163e12 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b133c7ec85..049c8243f6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -597,7 +597,6 @@ int sqlite3VdbeExec( } assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); -// p->rc = SQLITE_OK; p->iCurrentTime = 0; assert( p->explain==0 ); p->pResultSet = 0; diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh index 8dc852bef5..5f2c8345be 100644 --- a/tool/kvtest-speed.sh +++ b/tool/kvtest-speed.sh @@ -21,11 +21,10 @@ gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest # First run using SQL rm cachegrind.out.[1-9][0-9]* valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt -mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME -cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-sql-$NAME.txt +mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME +cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt # Second run using the sqlite3_blob object -rm cachegrind.out.[1-9][0-9]* valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt From 610e17bd91bca66ec7c03d7e60c2024e32ac989b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 31 Jan 2017 17:31:30 +0000 Subject: [PATCH 091/292] Add a speed-test program for the sessions module. FossilOrigin-Name: 25f1275fe3e940c1d9a7b013cb3744304b2eda1e --- ext/session/session_speed_test.c | 360 +++++++++++++++++++++++++++++++ manifest | 11 +- manifest.uuid | 2 +- 3 files changed, 367 insertions(+), 6 deletions(-) create mode 100644 ext/session/session_speed_test.c diff --git a/ext/session/session_speed_test.c b/ext/session/session_speed_test.c new file mode 100644 index 0000000000..14b97ea07b --- /dev/null +++ b/ext/session/session_speed_test.c @@ -0,0 +1,360 @@ +/* +** 2017 January 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the source code for a standalone program used to +** test the performance of the sessions module. Compile and run: +** +** ./session_speed_test -help +** +** for details. +*/ + +#include "sqlite3.h" +#include +#include +#include +#include +#include + +/************************************************************************* +** Start of generic command line parser. +*/ +#define CMDLINE_BARE 0 +#define CMDLINE_INTEGER 1 +#define CMDLINE_STRING 2 +#define CMDLINE_BOOLEAN 3 + +typedef struct CmdLineOption CmdLineOption; +struct CmdLineOption { + const char *zText; /* Name of command line option */ + const char *zHelp; /* Help text for option */ + int eType; /* One of the CMDLINE_* values */ + int iOff; /* Offset of output variable */ +}; + +#define CMDLINE_INT32(x,y,z) {x, y, CMDLINE_INTEGER, z} +#define CMDLINE_BOOL(x,y,z) {x, y, CMDLINE_BOOLEAN, z} +#define CMDLINE_TEXT(x,y,z) {x, y, CMDLINE_STRING, z} +#define CMDLINE_NONE(x,y,z) {x, y, CMDLINE_BARE, z} + +static void option_requires_argument_error(CmdLineOption *pOpt){ + fprintf(stderr, "Option requires a%s argument: %s\n", + pOpt->eType==CMDLINE_INTEGER ? "n integer" : + pOpt->eType==CMDLINE_STRING ? " string" : " boolean", + pOpt->zText + ); + exit(1); +} + +static void ambiguous_option_error(const char *zArg){ + fprintf(stderr, "Option is ambiguous: %s\n", zArg); + exit(1); +} + +static void unknown_option_error( + const char *zArg, + CmdLineOption *aOpt, + const char *zHelp +){ + int i; + fprintf(stderr, "Unknown option: %s\n", zArg); + fprintf(stderr, "\nOptions are:\n"); + fprintf(stderr, " % -30sEcho command line options\n", "-cmdline:verbose"); + for(i=0; aOpt[i].zText; i++){ + int eType = aOpt[i].eType; + char *zOpt = sqlite3_mprintf("%s %s", aOpt[i].zText, + eType==CMDLINE_BARE ? "" : + eType==CMDLINE_INTEGER ? "N" : + eType==CMDLINE_BOOLEAN ? "BOOLEAN" : "TEXT" + ); + fprintf(stderr, " % -30s%s\n", zOpt, aOpt[i].zHelp); + sqlite3_free(zOpt); + } + if( zHelp ){ + fprintf(stderr, "\n%s\n", zHelp); + } + exit(1); +} + +static int get_integer_option(CmdLineOption *pOpt, const char *zArg){ + int i = 0; + int iRet = 0; + int bSign = 1; + if( zArg[0]=='-' ){ + bSign = -1; + i = 1; + } + while( zArg[i] ){ + if( zArg[i]<'0' || zArg[i]>'9' ) option_requires_argument_error(pOpt); + iRet = iRet*10 + (zArg[i] - '0'); + i++; + } + return (iRet*bSign); +} + +static int get_boolean_option(CmdLineOption *pOpt, const char *zArg){ + if( 0==sqlite3_stricmp(zArg, "true") ) return 1; + if( 0==sqlite3_stricmp(zArg, "1") ) return 1; + if( 0==sqlite3_stricmp(zArg, "0") ) return 0; + if( 0==sqlite3_stricmp(zArg, "false") ) return 0; + option_requires_argument_error(pOpt); + return 0; +} + +static void parse_command_line( + int argc, + char **argv, + int iStart, + CmdLineOption *aOpt, + void *pStruct, + const char *zHelp +){ + char *pOut = (char*)pStruct; + int bVerbose = 0; + int iArg; + + for(iArg=iStart; iArgzText, zArg, nArg) ){ + if( nMatch ){ + ambiguous_option_error(zArg); + } + nMatch++; + if( pOpt->eType==CMDLINE_BARE ){ + *(int*)(&pOut[pOpt->iOff]) = 1; + }else{ + iArg++; + if( iArg==argc ){ + option_requires_argument_error(pOpt); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + *(int*)(&pOut[pOpt->iOff]) = get_integer_option(pOpt, argv[iArg]); + break; + case CMDLINE_STRING: + *(const char**)(&pOut[pOpt->iOff]) = argv[iArg]; + break; + case CMDLINE_BOOLEAN: + *(int*)(&pOut[pOpt->iOff]) = get_boolean_option(pOpt, argv[iArg]); + break; + } + } + } + } + + if( nMatch==0 && 0==sqlite3_strnicmp("-cmdline:verbose", zArg, nArg) ){ + bVerbose = 1; + nMatch = 1; + } + + if( nMatch==0 ){ + unknown_option_error(zArg, aOpt, zHelp); + } + } + + if( bVerbose ){ + int iOpt; + fprintf(stdout, "Options are: "); + for(iOpt=0; aOpt[iOpt].zText; iOpt++){ + CmdLineOption *pOpt = &aOpt[iOpt]; + if( pOpt->eType!=CMDLINE_BARE || *(int*)(&pOut[pOpt->iOff]) ){ + fprintf(stdout, "%s ", pOpt->zText); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_BOOLEAN: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_STRING: + fprintf(stdout, "%s ", *(const char**)(&pOut[pOpt->iOff])); + break; + } + } + fprintf(stdout, "\n"); + } +} +/* +** End of generic command line parser. +*************************************************************************/ + +static void abort_due_to_error(int rc){ + fprintf(stderr, "Error: %d\n"); + exit(-1); +} + +static void execsql(sqlite3 *db, const char *zSql){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); +} + +static int xConflict(void *pCtx, int eConflict, sqlite3_changeset_iter *p){ + return SQLITE_CHANGESET_ABORT; +} + +static void run_test( + sqlite3 *db, + sqlite3 *db2, + int nRow, + const char *zSql +){ + sqlite3_session *pSession = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int i; + int nChangeset; + void *pChangeset; + + /* Attach a session object to database db */ + rc = sqlite3session_create(db, "main", &pSession); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Configure the session to capture changes on all tables */ + rc = sqlite3session_attach(pSession, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Prepare the SQL statement */ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Open a transaction */ + execsql(db, "BEGIN"); + + /* Execute the SQL statement nRow times */ + for(i=0; i Date: Tue, 31 Jan 2017 19:02:15 +0000 Subject: [PATCH 092/292] Simplifications to blobSeekToRow(). FossilOrigin-Name: 495ea824093ff535734c22c3115384c08f855c02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeblob.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 3c10f176f0..fa96e1cee3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sC99-style\scomment.\s\sFixes\sto\sthe\skvtest-speed.sh\sscript. -D 2017-01-31T16:49:01.913 +C Simplifications\sto\sblobSeekToRow(). +D 2017-01-31T19:02:15.083 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -463,7 +463,7 @@ F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 281cb70332dc8b593b8c7afe776f3a2ba7d4255e F src/vdbeapi.c 7a65f10684982daecfce50f557f2632b7f20b198 F src/vdbeaux.c 6847b02aa2db536ed15d90f1fdc2923afef93c5b -F src/vdbeblob.c 2b3d1ad915dbe5dc92c48759dc18fa8c697e78e5 +F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2361b03b61311aab9b9ec9de040bbb73be31be0d -R 085c25b2bd73ed43799df664becffebf +P 91eb6b628e278d20eccc647293e5b30765163e12 +R 793d15c782afd1bb1c67c16147a858a7 U drh -Z 9d1629688efb4c919cbb0184aee51efa +Z 0d3326b5592bbe49f8801dfaea3a7026 diff --git a/manifest.uuid b/manifest.uuid index 80341cb011..fb73fe6210 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91eb6b628e278d20eccc647293e5b30765163e12 \ No newline at end of file +495ea824093ff535734c22c3115384c08f855c02 \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 810f78860f..5692c1c606 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -55,7 +55,6 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ int rc; /* Error code */ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; - sqlite3 *db = v->db; /* Set the value of register r[1] in the SQL statement to integer iRow. ** This is done directly as a performance optimization @@ -69,9 +68,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ ** counter is faster. */ if( v->pc>3 ){ v->pc = 3; - db->nVdbeExec++; - rc = sqlite3VdbeExec((Vdbe*)p->pStmt); - db->nVdbeExec--; + rc = sqlite3VdbeExec(v); }else{ rc = sqlite3_step(p->pStmt); } From fcbd6cdb41a8f338ed3c953405b4256cdfaba6c2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Jan 2017 21:22:03 +0000 Subject: [PATCH 093/292] R-TREE optimization: unwrap the coordinate decode loop in rtreeCallbackConstraint(). FossilOrigin-Name: 0bf7b51896ec441f62490964c7a44a3c75c6b7e2 --- ext/rtree/rtree.c | 10 +++++++--- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index c7de6ac123..d69c966ddd 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -966,10 +966,14 @@ static int rtreeCallbackConstraint( if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ pInfo->iRowid = readInt64(pCellData); } - pCellData += 8; - for(i=0; i=2 && (nCoord&1)==0 ); + i = 0; + do{ + pCellData += 8; RTREE_DECODE_COORD(eInt, pCellData, aCoord[i]); - } + RTREE_DECODE_COORD(eInt, (pCellData+4), aCoord[i+1]); + i+= 2; + }while( iop==RTREE_MATCH ){ rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, nCoord, aCoord, &i); diff --git a/manifest b/manifest index a8c339e791..cb6735cb6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\ssmall\sperformance\simprovements\sand\ssize\sreductions\sin\s\nsqlite3VdbeExec()\sand\sblobSeekToRow(). -D 2017-01-31T19:10:42.671 +C R-TREE\soptimization:\sunwrap\sthe\scoordinate\sdecode\sloop\sin\nrtreeCallbackConstraint(). +D 2017-01-31T21:22:03.245 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 078fc27417b48aaebe988621bf96ef0e1645b201 +F ext/rtree/rtree.c 81fea8b100c6195f8a42119109d6ec2016e613f3 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1549,8 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b63deed600b1a457a6960ebad5645f4de9c56e5d 495ea824093ff535734c22c3115384c08f855c02 -R 793d15c782afd1bb1c67c16147a858a7 -T +closed 495ea824093ff535734c22c3115384c08f855c02 +P 85dddf2b453b8afaf1f485b96084d31e22f97dda +R 80f20a76d631de95caa5095989ae297f U drh -Z c7f2b5c04f4a8cad3ed5cf73468bb400 +Z d89e7b06786861fbe2704ca06b1411f8 diff --git a/manifest.uuid b/manifest.uuid index 321250201a..0415fad4fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85dddf2b453b8afaf1f485b96084d31e22f97dda \ No newline at end of file +0bf7b51896ec441f62490964c7a44a3c75c6b7e2 \ No newline at end of file From 35497fccbe358dda7b95e1a4c109020725877f0b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 01:34:15 +0000 Subject: [PATCH 094/292] Improved comments on the statGet() implementation in ANALYZE. No changes to code. FossilOrigin-Name: 9663eea2a16bb4eec71476d307a3722a768308c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cb6735cb6d..cd07701b23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C R-TREE\soptimization:\sunwrap\sthe\scoordinate\sdecode\sloop\sin\nrtreeCallbackConstraint(). -D 2017-01-31T21:22:03.245 +C Improved\scomments\son\sthe\sstatGet()\simplementation\sin\sANALYZE.\s\sNo\schanges\nto\scode. +D 2017-02-01T01:34:15.043 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -328,7 +328,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 317dbaf31c16050582b09bf4f323b4e0f1813251 +F src/analyze.c ac7a5d7e3cee07d074697904e00e4a8ab7b2b4f5 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85dddf2b453b8afaf1f485b96084d31e22f97dda -R 80f20a76d631de95caa5095989ae297f +P 0bf7b51896ec441f62490964c7a44a3c75c6b7e2 +R bb550435b988e435dcc2fd1779602487 U drh -Z d89e7b06786861fbe2704ca06b1411f8 +Z 36c251bda157d6101f2fb97842697f89 diff --git a/manifest.uuid b/manifest.uuid index 0415fad4fb..1081a26f59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0bf7b51896ec441f62490964c7a44a3c75c6b7e2 \ No newline at end of file +9663eea2a16bb4eec71476d307a3722a768308c3 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c079132f4a..890ae7c3b2 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -797,6 +797,12 @@ static const FuncDef statPushFuncdef = { ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a poiner to a Stat4Accum object, never a +** NULL. +** ** If neither STAT3 nor STAT4 are enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the From 14494fa712d71370eb92e15c4e4db20534fcd832 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 02:25:28 +0000 Subject: [PATCH 095/292] More RTREE performance optimizations related to decoding values. FossilOrigin-Name: c5395e7496d0cd593f5e16ee5f6719d020dc0c66 --- ext/rtree/rtree.c | 18 +++++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index d69c966ddd..f1db60b22f 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -370,12 +370,20 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 + memcpy(&pCoord->u, p, 4); + pCoord->u = ((pCoord->u>>24)&0xff)|((pCoord->u>>8)&0xff00)| + ((pCoord->u&0xff)<<24)|((pCoord->u&0xff00)<<8); +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 + memcpy(&pCoord->u, p, 4); +#else pCoord->u = ( (((u32)p[0]) << 24) + (((u32)p[1]) << 16) + (((u32)p[2]) << 8) + (((u32)p[3]) << 0) ); +#endif } static i64 readInt64(u8 *p){ return ( @@ -736,13 +744,21 @@ static void nodeGetCell( ){ u8 *pData; RtreeCoord *pCoord; - int ii; + int ii = 0; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); pCoord = pCell->aCoord; + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim*2 ); +#if 0 for(ii=0; iinDim*2; ii++){ readCoord(&pData[ii*4], &pCoord[ii]); } +#endif } diff --git a/manifest b/manifest index cd07701b23..3a5177423f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sthe\sstatGet()\simplementation\sin\sANALYZE.\s\sNo\schanges\nto\scode. -D 2017-02-01T01:34:15.043 +C More\sRTREE\sperformance\soptimizations\srelated\sto\sdecoding\svalues. +D 2017-02-01T02:25:28.535 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 81fea8b100c6195f8a42119109d6ec2016e613f3 +F ext/rtree/rtree.c f66b3d232ea98285548107caaac55f110a0b6709 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1549,7 +1549,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0bf7b51896ec441f62490964c7a44a3c75c6b7e2 -R bb550435b988e435dcc2fd1779602487 +P 9663eea2a16bb4eec71476d307a3722a768308c3 +R 5d2055e5a67aa8455e161eeb4945e6a1 U drh -Z 36c251bda157d6101f2fb97842697f89 +Z 3923bc85bfbb1c570a9230cd663fc66b diff --git a/manifest.uuid b/manifest.uuid index 1081a26f59..becf54d88b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9663eea2a16bb4eec71476d307a3722a768308c3 \ No newline at end of file +c5395e7496d0cd593f5e16ee5f6719d020dc0c66 \ No newline at end of file From f6c69221e3b5f48fceb5e3664746716ea7e31f46 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Feb 2017 14:19:43 +0000 Subject: [PATCH 096/292] Update the documentation comment in sqlite.h.in for sqlite3_preupdate_hook(). FossilOrigin-Name: 7f8570208c06c056d426e9299d9930181a0464f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 24 ++++++++++++++---------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 01669f9e0e..b3f47391f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sso\sthat\sthe\spre-update\shook\sand\sthe\ssessions\smodule\swork\swith\sWITHOUT\nROWID\stables. -D 2017-02-01T14:10:24.369 +C Update\sthe\sdocumentation\scomment\sin\ssqlite.h.in\sfor\ssqlite3_preupdate_hook(). +D 2017-02-01T14:19:43.743 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -395,7 +395,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 -F src/sqlite.h.in 8fd2b1a4e4aac023d4533313442528b81105fab3 +F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h 6f29b23415ce207c2fda61054bf244b74f96359c @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c5395e7496d0cd593f5e16ee5f6719d020dc0c66 25f1275fe3e940c1d9a7b013cb3744304b2eda1e -R f7a2435ee7007416912dcdfda9016a2c +P 964bdc27f8f1b1db2e5c0c2a65c8156614cbe087 +R c3a8e9059dd92e8c282633700ca50a30 U dan -Z 84db606ff358cb3acbf24f0cef50dec1 +Z fded3ef392ff672f8205c955a05c2af4 diff --git a/manifest.uuid b/manifest.uuid index f45e6e6220..dda985aaff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -964bdc27f8f1b1db2e5c0c2a65c8156614cbe087 \ No newline at end of file +7f8570208c06c056d426e9299d9930181a0464f8 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b0ec7f7e15..eaa75fc249 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8180,7 +8180,7 @@ int sqlite3_db_cacheflush(sqlite3*); ** ** ^The [sqlite3_preupdate_hook()] interface registers a callback function ** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation -** on a [rowid table]. +** on a database table. ** ^At most one preupdate hook may be registered at a time on a single ** [database connection]; each call to [sqlite3_preupdate_hook()] overrides ** the previous setting. @@ -8189,9 +8189,9 @@ int sqlite3_db_cacheflush(sqlite3*); ** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as ** the first parameter to callbacks. ** -** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate -** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID] -** tables. +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_master or sqlite_stat1. ** ** ^The second parameter to the preupdate callback is a pointer to ** the [database connection] that registered the preupdate hook. @@ -8205,12 +8205,16 @@ int sqlite3_db_cacheflush(sqlite3*); ** databases.)^ ** ^The fifth parameter to the preupdate callback is the name of the ** table that is being modified. -** ^The sixth parameter to the preupdate callback is the initial [rowid] of the -** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is -** undefined for SQLITE_INSERT changes. -** ^The seventh parameter to the preupdate callback is the final [rowid] of -** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is -** undefined for SQLITE_DELETE changes. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** INSERT operations on rowid tables. ** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces From 1650fcb1b5627ec7ab2a514b9a42920e8e739ac7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 15:19:29 +0000 Subject: [PATCH 097/292] Fix the build by making the OPFLAG_ISNOOP macro available unconditionally. FossilOrigin-Name: 510933cb24c5bf883265af3a6075e60a4b5ffa37 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b3f47391f1..8145330051 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\scomment\sin\ssqlite.h.in\sfor\ssqlite3_preupdate_hook(). -D 2017-02-01T14:19:43.743 +C Fix\sthe\sbuild\sby\smaking\sthe\sOPFLAG_ISNOOP\smacro\savailable\sunconditionally. +D 2017-02-01T15:19:29.202 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -398,7 +398,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 6f29b23415ce207c2fda61054bf244b74f96359c +F src/sqliteInt.h 3724c48e82605b471bbf7e41109b5850c6e89f31 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 964bdc27f8f1b1db2e5c0c2a65c8156614cbe087 -R c3a8e9059dd92e8c282633700ca50a30 -U dan -Z fded3ef392ff672f8205c955a05c2af4 +P 7f8570208c06c056d426e9299d9930181a0464f8 +R bbb1f490f7311ec1c087c3ef4607e5e0 +U drh +Z 645035379b970696e5aed344f8a4be19 diff --git a/manifest.uuid b/manifest.uuid index dda985aaff..0823e8bab6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f8570208c06c056d426e9299d9930181a0464f8 \ No newline at end of file +510933cb24c5bf883265af3a6075e60a4b5ffa37 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4a28ebcd5a..3faf9d3982 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3057,9 +3057,7 @@ struct AuthContext { #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ -#endif #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ From 03626e38127d4d0667eb23243c479719c1d30b9a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 15:24:32 +0000 Subject: [PATCH 098/292] Use compiler intrinsic functions (when available) for byteswapping in RTREE. FossilOrigin-Name: 82fcd54a5941c20895ffc22d8009c1ebdae44eda --- ext/rtree/rtree.c | 142 ++++++++++++++++++++++++++++++++++++++++------ manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 131 insertions(+), 25 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index f1db60b22f..637fb79d05 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -362,6 +362,65 @@ struct RtreeMatchArg { # define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif +/* What version of GCC is being used. 0 means GCC is not being used */ +#ifndef GCC_VERSION +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* What version of CLANG is being used. 0 means CLANG is not being used */ +#ifndef CLANG_VERSION +#if defined(__clang__) && !defined(_WIN32) +# define CLANG_VERSION \ + (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) +#else +# define CLANG_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGMATION +# define testcase(X) +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 1234 +#endif +#if (defined(sparc) || defined(__ppc__)) \ + && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 4321 +#endif +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + /* ** Functions to deserialize a 16 bit integer, 32 bit real number and ** 64 bit integer. The deserialized value is returned. @@ -370,12 +429,16 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 - memcpy(&pCoord->u, p, 4); + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 pCoord->u = ((pCoord->u>>24)&0xff)|((pCoord->u>>8)&0xff00)| ((pCoord->u&0xff)<<24)|((pCoord->u&0xff00)<<8); -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 - memcpy(&pCoord->u, p, 4); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; #else pCoord->u = ( (((u32)p[0]) << 24) + @@ -386,6 +449,20 @@ static void readCoord(u8 *p, RtreeCoord *pCoord){ #endif } static i64 readInt64(u8 *p){ + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else return ( (((i64)p[0]) << 56) + (((i64)p[1]) << 48) + @@ -396,6 +473,7 @@ static i64 readInt64(u8 *p){ (((i64)p[6]) << 8) + (((i64)p[7]) << 0) ); +#endif } /* @@ -410,16 +488,38 @@ static int writeInt16(u8 *p, int i){ } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else i = pCoord->u; p[0] = (i>>24)&0xFF; p[1] = (i>>16)&0xFF; p[2] = (i>> 8)&0xFF; p[3] = (i>> 0)&0xFF; +#endif return 4; } static int writeInt64(u8 *p, i64 i){ + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* Not always 8-byte aligned */ +#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else p[0] = (i>>56)&0xFF; p[1] = (i>>48)&0xFF; p[2] = (i>>40)&0xFF; @@ -428,6 +528,7 @@ static int writeInt64(u8 *p, i64 i){ p[5] = (i>>16)&0xFF; p[6] = (i>> 8)&0xFF; p[7] = (i>> 0)&0xFF; +#endif return 8; } @@ -754,11 +855,6 @@ static void nodeGetCell( pData += 8; ii += 2; }while( iinDim*2 ); -#if 0 - for(ii=0; iinDim*2; ii++){ - readCoord(&pData[ii*4], &pCoord[ii]); - } -#endif } @@ -927,15 +1023,22 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ** false. a[] is the four bytes of the on-disk record to be decoded. ** Store the results in "r". ** -** There are three versions of this macro, one each for little-endian and -** big-endian processors and a third generic implementation. The endian- -** specific implementations are much faster and are preferred if the -** processor endianness is known at compile-time. The SQLITE_BYTEORDER -** macro is part of sqliteInt.h and hence the endian-specific -** implementation will only be used if this module is compiled as part -** of the amalgamation. +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. */ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -943,7 +1046,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#elif SQLITE_BYTEORDER==4321 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -986,6 +1089,7 @@ static int rtreeCallbackConstraint( i = 0; do{ pCellData += 8; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, aCoord[i]); RTREE_DECODE_COORD(eInt, (pCellData+4), aCoord[i+1]); i+= 2; @@ -1029,6 +1133,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ case RTREE_LE: case RTREE_LT: @@ -1069,6 +1174,7 @@ static void rtreeLeafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); pCellData += 8 + p->iCoord*4; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_LE: if( xN <= p->u.rValue ) return; break; diff --git a/manifest b/manifest index 8145330051..bc185a2197 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sby\smaking\sthe\sOPFLAG_ISNOOP\smacro\savailable\sunconditionally. -D 2017-02-01T15:19:29.202 +C Use\scompiler\sintrinsic\sfunctions\s(when\savailable)\sfor\sbyteswapping\sin\sRTREE. +D 2017-02-01T15:24:32.835 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c f66b3d232ea98285548107caaac55f110a0b6709 +F ext/rtree/rtree.c d8ef14e964a9390197ba7e511aa47b89dc39416c F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7f8570208c06c056d426e9299d9930181a0464f8 -R bbb1f490f7311ec1c087c3ef4607e5e0 +P 510933cb24c5bf883265af3a6075e60a4b5ffa37 +R 13778722a0ec2443f74339612adbbcf9 U drh -Z 645035379b970696e5aed344f8a4be19 +Z b1e5b219635e474580d8132f194fff51 diff --git a/manifest.uuid b/manifest.uuid index 0823e8bab6..6e2e6a9d62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -510933cb24c5bf883265af3a6075e60a4b5ffa37 \ No newline at end of file +82fcd54a5941c20895ffc22d8009c1ebdae44eda \ No newline at end of file From 0e6f67b7545d37f47fa5bdd6907c857a73da3e57 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 15:49:02 +0000 Subject: [PATCH 099/292] Precompute the nDim2 value in the Rtree object and use that to make loops over coordinates faster. FossilOrigin-Name: f1f3c8cc733a05c12dd980f2dfa0ab4ccd76c04b --- ext/rtree/rtree.c | 43 +++++++++++++++++++++++++------------------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 637fb79d05..7e8b7b2b48 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -116,6 +116,7 @@ struct Rtree { sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ @@ -711,7 +712,7 @@ static void nodeOverwriteCell( int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; p += writeInt64(p, pCell->iRowid); - for(ii=0; ii<(pRtree->nDim*2); ii++){ + for(ii=0; iinDim2; ii++){ p += writeCoord(p, &pCell->aCoord[ii]); } pNode->isDirty = 1; @@ -854,7 +855,7 @@ static void nodeGetCell( readCoord(pData+4, &pCoord[ii+1]); pData += 8; ii += 2; - }while( iinDim*2 ); + }while( iinDim2 ); } @@ -1712,7 +1713,7 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } - p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ @@ -1878,10 +1879,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** Return the N-dimensional volumn of the cell stored in *p. */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ - RtreeDValue area = (RtreeDValue)1; + RtreeDValue area; int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + area = DCOORD(p->aCoord[1]) - DCOORD(p->aCoord[0]); + for(ii=2; iinDim2; ii+=2){ + area *= DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]); } return area; } @@ -1891,9 +1893,10 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ ** of the objects size in each dimension. */ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin = (RtreeDValue)0; + RtreeDValue margin; int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + margin = DCOORD(p->aCoord[1]) - DCOORD(p->aCoord[0]); + for(ii=2; iinDim2; ii+=2){ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; @@ -1903,17 +1906,19 @@ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ ** Store the union of cells p1 and p2 in p1. */ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; + int ii = 0; if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); - } + ii += 2; + }while( iinDim2 ); }else{ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); - } + ii += 2; + }while( iinDim2 ); } } @@ -1924,7 +1929,7 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iinDim2; ii+=2){ RtreeCoord *a1 = &p1->aCoord[ii]; RtreeCoord *a2 = &p2->aCoord[ii]; if( (!isInt && (a2[0].fa1[1].f)) @@ -1959,7 +1964,7 @@ static RtreeDValue cellOverlap( for(ii=0; iinDim*2); jj+=2){ + for(jj=0; jjnDim2; jj+=2){ RtreeDValue x1, x2; x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); @@ -3015,7 +3020,7 @@ static int rtreeUpdate( ** This problem was discovered after years of use, so we silently ignore ** these kinds of misdeclared tables to avoid breaking any legacy. */ - assert( nData<=(pRtree->nDim*2 + 3) ); + assert( nData<=(pRtree->nDim2 + 3) ); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -3393,7 +3398,8 @@ static int rtreeInit( pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; pRtree->nDim = (argc-4)/2; - pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; + pRtree->nDim2 = argc - 4; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; pRtree->eCoordType = eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); @@ -3468,6 +3474,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); tree.nDim = sqlite3_value_int(apArg[0]); + tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); @@ -3480,7 +3487,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nodeGetCell(&tree, &node, ii, &cell); sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); nCell = (int)strlen(zCell); - for(jj=0; jj Date: Wed, 1 Feb 2017 16:41:30 +0000 Subject: [PATCH 100/292] Completely unroll the dimension loop inside of cellArea() in RTREE. FossilOrigin-Name: 3c4c0126c287f844220b65e00fec17c059fbb7c8 --- ext/rtree/rtree.c | 26 +++++++++++++++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 7e8b7b2b48..4d67d67044 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1879,11 +1879,27 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** Return the N-dimensional volumn of the cell stored in *p. */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ - RtreeDValue area; - int ii; - area = DCOORD(p->aCoord[1]) - DCOORD(p->aCoord[0]); - for(ii=2; iinDim2; ii+=2){ - area *= DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]); + RtreeDValue area = (RtreeDValue)1; + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].i - p->aCoord[8].i; + case 4: area *= p->aCoord[7].i - p->aCoord[6].i; + case 3: area *= p->aCoord[5].i - p->aCoord[4].i; + case 2: area *= p->aCoord[3].i - p->aCoord[2].i; + default: area *= p->aCoord[1].i - p->aCoord[0].i; + } } return area; } diff --git a/manifest b/manifest index 47d8c1d927..62887ba814 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Precompute\sthe\snDim2\svalue\sin\sthe\sRtree\sobject\sand\suse\sthat\sto\smake\sloops\nover\scoordinates\sfaster. -D 2017-02-01T15:49:02.390 +C Completely\sunroll\sthe\sdimension\sloop\sinside\sof\scellArea()\sin\sRTREE. +D 2017-02-01T16:41:30.471 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c bc73bae0af0d44f92b7d6cb513b719c1e89a7a20 +F ext/rtree/rtree.c 60df707ce5ae232351fc7fc8e916738b1a06bf26 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 82fcd54a5941c20895ffc22d8009c1ebdae44eda -R 2d19b39f1fe0fa1365532dfac9c432a7 +P f1f3c8cc733a05c12dd980f2dfa0ab4ccd76c04b +R f244aabca54009b102530102725b0e43 U drh -Z 1d3fc5da859da34aeaaaf616405346d2 +Z 396b9b4a99bf06a5c73ba01696403f75 diff --git a/manifest.uuid b/manifest.uuid index 5d3a3937af..a4a1237e47 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1f3c8cc733a05c12dd980f2dfa0ab4ccd76c04b \ No newline at end of file +3c4c0126c287f844220b65e00fec17c059fbb7c8 \ No newline at end of file From 31a13495153ece728af9641e4327f009088cab3f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Feb 2017 17:08:56 +0000 Subject: [PATCH 101/292] Unwind the RTREE dimension loop inside of rtreeCallbackConstraint(). FossilOrigin-Name: 4854ea9c18e7d8066c90b41568d0fae97b01ea6d --- ext/rtree/rtree.c | 41 ++++++++++++++++++++++++++++++++--------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 4d67d67044..ac76c18485 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1078,6 +1078,7 @@ static int rtreeCallbackConstraint( sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ int nCoord = pInfo->nCoord; /* No. of coordinates */ int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); @@ -1086,15 +1087,37 @@ static int rtreeCallbackConstraint( if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ pInfo->iRowid = readInt64(pCellData); } - assert( nCoord>=2 && (nCoord&1)==0 ); - i = 0; - do{ - pCellData += 8; - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ - RTREE_DECODE_COORD(eInt, pCellData, aCoord[i]); - RTREE_DECODE_COORD(eInt, (pCellData+4), aCoord[i+1]); - i+= 2; - }while( iop==RTREE_MATCH ){ rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, nCoord, aCoord, &i); diff --git a/manifest b/manifest index 62887ba814..8bb675991a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Completely\sunroll\sthe\sdimension\sloop\sinside\sof\scellArea()\sin\sRTREE. -D 2017-02-01T16:41:30.471 +C Unwind\sthe\sRTREE\sdimension\sloop\sinside\sof\srtreeCallbackConstraint(). +D 2017-02-01T17:08:56.249 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 60df707ce5ae232351fc7fc8e916738b1a06bf26 +F ext/rtree/rtree.c f2c8604fcdbe13f960c5f0b97acbc2a870b72bf2 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f1f3c8cc733a05c12dd980f2dfa0ab4ccd76c04b -R f244aabca54009b102530102725b0e43 +P 3c4c0126c287f844220b65e00fec17c059fbb7c8 +R f35bf88856ef1a64759d9b647928073d U drh -Z 396b9b4a99bf06a5c73ba01696403f75 +Z 9add4b4d28dce63431ad3a0684bb5601 diff --git a/manifest.uuid b/manifest.uuid index a4a1237e47..8728c8d9a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c4c0126c287f844220b65e00fec17c059fbb7c8 \ No newline at end of file +4854ea9c18e7d8066c90b41568d0fae97b01ea6d \ No newline at end of file From 8c5e8feb35641dc8fc8b3512aa79fd0fcfff775e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 1 Feb 2017 22:32:49 +0000 Subject: [PATCH 102/292] Add an option to the MSVC makefile to enable treating warnings as errors. FossilOrigin-Name: 6a378c29b43d61313ca9daa599e59d8eeeed9a27 --- Makefile.msc | 13 +++++++++++++ autoconf/Makefile.msc | 13 +++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index d4aba301fb..4a29f913c8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -24,6 +24,13 @@ USE_AMALGAMATION = 1 USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + # Set this non-0 to enable full runtime error checks (-RTC1, etc). This # has no effect if (any) optimizations are enabled. # @@ -493,6 +500,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index b53e2370eb..32d03dbf35 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -24,6 +24,13 @@ TOP = . USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + # Set this non-0 to enable full runtime error checks (-RTC1, etc). This # has no effect if (any) optimizations are enabled. # @@ -454,6 +461,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) diff --git a/manifest b/manifest index 8bb675991a..2bea70b5b4 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Unwind\sthe\sRTREE\sdimension\sloop\sinside\sof\srtreeCallbackConstraint(). -D 2017-02-01T17:08:56.249 +C Add\san\soption\sto\sthe\sMSVC\smakefile\sto\senable\streating\swarnings\sas\serrors. +D 2017-02-01T22:32:49.021 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da +F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION cddd8d88dc8202afa0ebc96da61fc4acbd1e96a5 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc b6d27f735911fd09a589d3c966936c47a5850c17 +F autoconf/Makefile.msc 3f29e0fc2a78e113250d79a5da375c1d8d8c06b0 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c4c0126c287f844220b65e00fec17c059fbb7c8 -R f35bf88856ef1a64759d9b647928073d -U drh -Z 9add4b4d28dce63431ad3a0684bb5601 +P 4854ea9c18e7d8066c90b41568d0fae97b01ea6d +R edeb8ee1db3973795f07ac14a992bc66 +U mistachkin +Z 3bfb007e12a85d09e541803b4d02a604 diff --git a/manifest.uuid b/manifest.uuid index 8728c8d9a3..72864f657b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4854ea9c18e7d8066c90b41568d0fae97b01ea6d \ No newline at end of file +6a378c29b43d61313ca9daa599e59d8eeeed9a27 \ No newline at end of file From 2e525322172e49823cc72daa0e84e5a84b5d2f62 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 1 Feb 2017 22:43:08 +0000 Subject: [PATCH 103/292] Fix harmless compiler warnings seen with MSVC. FossilOrigin-Name: 997f765bc6706769ae15f3e719354473e02bd78b --- ext/fts3/fts3.c | 8 ++++---- ext/rtree/rtree.c | 14 +++++++------- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/pragma.c | 2 +- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 748faefec5..c12e3d2156 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1363,9 +1363,9 @@ static int fts3InitVtab( p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; @@ -3422,7 +3422,7 @@ static int fts3SetHasStat(Fts3Table *p){ if( rc==SQLITE_OK ){ int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; } sqlite3_free(zSql); }else{ diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index ac76c18485..ac84562590 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1515,7 +1515,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ if( rScoreeWithin = eWithin; + p->eWithin = (u8)eWithin; p->id = x.id; p->iCell = x.iCell; RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); @@ -1703,7 +1703,7 @@ static int rtreeFilter( p->id = iNode; p->eWithin = PARTLY_WITHIN; rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); - p->iCell = iCell; + p->iCell = (u8)iCell; RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); }else{ pCsr->atEOF = 1; @@ -1751,7 +1751,7 @@ static int rtreeFilter( } if( rc==SQLITE_OK ){ RtreeSearchPoint *pNew; - pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); if( pNew==0 ) return SQLITE_NOMEM; pNew->id = 1; pNew->iCell = 0; @@ -1879,7 +1879,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -3436,10 +3436,10 @@ static int rtreeInit( pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; - pRtree->nDim = (argc-4)/2; + pRtree->nDim = (u8)((argc-4)/2); pRtree->nDim2 = argc - 4; pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; - pRtree->eCoordType = eCoordType; + pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); @@ -3512,7 +3512,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); - tree.nDim = sqlite3_value_int(apArg[0]); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); diff --git a/manifest b/manifest index 2bea70b5b4..1d0d458233 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\soption\sto\sthe\sMSVC\smakefile\sto\senable\streating\swarnings\sas\serrors. -D 2017-02-01T22:32:49.021 +C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. +D 2017-02-01T22:43:08.270 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e028eb13432f108d2e22cded019fc980700e4e00 +F ext/fts3/fts3.c 5a8cafedffd101e9946f8909ecf8d34aaa383b4d F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 89d0bd4595a0de384dac78e94b803de12586e8dd F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c f2c8604fcdbe13f960c5f0b97acbc2a870b72bf2 +F ext/rtree/rtree.c 880edfc9a342c130fb2e220c4d7471729e764334 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -386,7 +386,7 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 51070ec9b8251bbf9c6ea3d35fd96a458752929e F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 0e7a7c6f1c6fd8ff50c0fff65b8bb80174bc49c5 +F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd @@ -1552,7 +1552,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4854ea9c18e7d8066c90b41568d0fae97b01ea6d -R edeb8ee1db3973795f07ac14a992bc66 +P 6a378c29b43d61313ca9daa599e59d8eeeed9a27 +R 1fd2d9b1d5e3d137e94e02ebe9288650 +T *branch * msvcWarn +T *sym-msvcWarn * +T -sym-trunk * U mistachkin -Z 3bfb007e12a85d09e541803b4d02a604 +Z c7bdc5431b2e37708dfabf3606031db2 diff --git a/manifest.uuid b/manifest.uuid index 72864f657b..365c5fd48f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a378c29b43d61313ca9daa599e59d8eeeed9a27 \ No newline at end of file +997f765bc6706769ae15f3e719354473e02bd78b \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 5251d0cd26..b1775a4082 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -279,7 +279,7 @@ const char *sqlite3JournalModename(int eMode){ ** Locate a pragma in the aPragmaName[] array. */ static const PragmaName *pragmaLocate(const char *zName){ - int upr, lwr, mid, rc; + int upr, lwr, mid = 0, rc; lwr = 0; upr = ArraySize(aPragmaName)-1; while( lwr<=upr ){ From 2fa517bf2a8dbc4e76464675a0e5ef5af776bfa3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 1 Feb 2017 22:59:29 +0000 Subject: [PATCH 104/292] Fix C99-style variable declaration issue seen with older versions of MSVC. FossilOrigin-Name: 54d285464a222c59327eb6c917c1cc0125a55a27 --- ext/rtree/rtree.c | 5 ++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index ac76c18485..fe89835972 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -450,20 +450,23 @@ static void readCoord(u8 *p, RtreeCoord *pCoord){ #endif } static i64 readInt64(u8 *p){ - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u64 x; + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return (i64)_byteswap_uint64(x); #elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) u64 x; + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return (i64)__builtin_bswap64(x); #elif SQLITE_BYTEORDER==4321 i64 x; + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return x; #else + testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ return ( (((i64)p[0]) << 56) + (((i64)p[1]) << 48) + diff --git a/manifest b/manifest index 2bea70b5b4..aa07fecd68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\soption\sto\sthe\sMSVC\smakefile\sto\senable\streating\swarnings\sas\serrors. -D 2017-02-01T22:32:49.021 +C Fix\sC99-style\svariable\sdeclaration\sissue\sseen\swith\solder\sversions\sof\sMSVC. +D 2017-02-01T22:59:29.767 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c f2c8604fcdbe13f960c5f0b97acbc2a870b72bf2 +F ext/rtree/rtree.c 58d3fa51e270b93e1d4ceb7ffb5c0f3b85cb4129 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4854ea9c18e7d8066c90b41568d0fae97b01ea6d -R edeb8ee1db3973795f07ac14a992bc66 +P 6a378c29b43d61313ca9daa599e59d8eeeed9a27 +R 464fa6ae427ab1455a9a5204291b6a50 U mistachkin -Z 3bfb007e12a85d09e541803b4d02a604 +Z 85455f002136d0ca47deec93d577c815 diff --git a/manifest.uuid b/manifest.uuid index 72864f657b..93c65ce658 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a378c29b43d61313ca9daa599e59d8eeeed9a27 \ No newline at end of file +54d285464a222c59327eb6c917c1cc0125a55a27 \ No newline at end of file From 5f7b95f73b9b6e157a9cfb90836dbac42cd4c97b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 1 Feb 2017 23:03:54 +0000 Subject: [PATCH 105/292] Backout the change in [02f6293f27] as it causes MSVC to complain. FossilOrigin-Name: aaae74d06f4865818465cfdb440258ae8a5b985a --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index aa07fecd68..257674eec1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sC99-style\svariable\sdeclaration\sissue\sseen\swith\solder\sversions\sof\sMSVC. -D 2017-02-01T22:59:29.767 +C Backout\sthe\schange\sin\s[02f6293f27]\sas\sit\scauses\sMSVC\sto\scomplain. +D 2017-02-01T23:03:54.479 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -460,7 +460,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c a88b0466fddf445ce752226d4698ca3faada620a F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c ec059d331da77cb2e3e8af3fb7e40443ed148fbd +F src/vdbe.c e7b1e860140f1d1803c6f4176b1e8f3801f3a290 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1552,7 +1552,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a378c29b43d61313ca9daa599e59d8eeeed9a27 -R 464fa6ae427ab1455a9a5204291b6a50 +P 54d285464a222c59327eb6c917c1cc0125a55a27 +Q -02f6293f278f7b0a0f4876f5c6a0f4dc42620d79 +R c167920e65eb1311d91ae6d6e9361130 U mistachkin -Z 85455f002136d0ca47deec93d577c815 +Z 464bc0b25a9ceacfcf91d7e8480e3c64 diff --git a/manifest.uuid b/manifest.uuid index 93c65ce658..6df479bdb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54d285464a222c59327eb6c917c1cc0125a55a27 \ No newline at end of file +aaae74d06f4865818465cfdb440258ae8a5b985a \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c8b3db8cf2..cb8a039abb 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -562,7 +562,7 @@ int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ Op *aOp = p->aOp; /* Copy of p->aOp */ - Op *pOp; /* Current operation */ + Op *pOp = aOp; /* Current operation */ #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) Op *pOrigOp; /* Value of pOp at the top of the loop */ #endif From 2343c7eb3f0f6fbb4d3719a09ac31bfb33a357d4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 00:46:55 +0000 Subject: [PATCH 106/292] This is an experimental patch that ensures that all cursors have their position saved prior to starting a ROLLBACK TO. FossilOrigin-Name: 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 --- manifest | 18 ++++++++++-------- manifest.uuid | 2 +- src/btree.c | 7 ++++++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6459c8e5ad..4cc36f422d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2017-02-01T23:06:17.747 +C This\sis\san\sexperimental\spatch\sthat\sensures\sthat\sall\scursors\shave\stheir\sposition\nsaved\sprior\sto\sstarting\sa\sROLLBACK\sTO. +D 2017-02-02T00:46:55.000 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -336,7 +336,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 6a63fa34e6fe86e87090e41963c0f2fcf9d3e16d +F src/btree.c 9fe65ab418d99e80289f024016b5e5e74c3059dd F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af @@ -1552,8 +1552,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aaae74d06f4865818465cfdb440258ae8a5b985a 997f765bc6706769ae15f3e719354473e02bd78b -R 939bce209d57b72db3d58f9894d61374 -T +closed 997f765bc6706769ae15f3e719354473e02bd78b -U mistachkin -Z c2a7081e9858115a573bd4695718af54 +P 0c66cf0f0a9ada2ddcb8d61001ef791b86226416 +R 329580c21139f900f64001f6acfc9557 +T *branch * savepoint-rollback +T *sym-savepoint-rollback * +T -sym-trunk * +U drh +Z 5087946e64391fb6ac59f69d722efdd6 diff --git a/manifest.uuid b/manifest.uuid index 0e34341685..9d5ad1c981 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c66cf0f0a9ada2ddcb8d61001ef791b86226416 \ No newline at end of file +01d97e5b6502b1811b52a681f445e1aaae6c0ee6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 45a1c5a0f8..de553423b8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4038,7 +4038,12 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); - rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ pBt->nPage = 0; From 6d683c5c6e7cfbd3393f64e6520caef5a5921a53 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 02:28:45 +0000 Subject: [PATCH 107/292] Use the sqlite3_blob interface for reading values from the %_node shadow table in RTREE. This is a work in progress. There are still some minor problems. FossilOrigin-Name: fc4917d730b29b0bf60fea5e0166728635783e9c --- ext/rtree/rtree.c | 106 ++++++++++++++++++++++++++++++++++------------ manifest | 18 ++++---- manifest.uuid | 2 +- 3 files changed, 88 insertions(+), 38 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 66cca987da..eae9ef819a 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -133,6 +133,9 @@ struct Rtree { RtreeNode *pDeleted; int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; + /* Statements to read/write/delete a record from xxx_node */ sqlite3_stmt *pReadNode; sqlite3_stmt *pWriteNode; @@ -615,6 +618,16 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ return pNode; } +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + if( pRtree->pNodeBlob ){ + sqlite3_blob_close(pRtree->pNodeBlob); + pRtree->pNodeBlob = 0; + } +} + /* ** Obtain a reference to an r-tree node. */ @@ -624,9 +637,8 @@ static int nodeAcquire( RtreeNode *pParent, /* Either the parent node or NULL */ RtreeNode **ppNode /* OUT: Acquired node */ ){ - int rc; - int rc2 = SQLITE_OK; - RtreeNode *pNode; + int rc = SQLITE_OK; + RtreeNode *pNode = 0; /* Check if the requested node is already in the hash table. If so, ** increase its reference count and return it. @@ -642,28 +654,42 @@ static int nodeAcquire( return SQLITE_OK; } - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); - rc = sqlite3_step(pRtree->pReadNode); - if( rc==SQLITE_ROW ){ - const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); - if( !pNode ){ - rc2 = SQLITE_NOMEM; - }else{ - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); - } + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return rc; + } + } + if( pRtree->pNodeBlob==0 ){ + char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); + if( zTab==0 ) return SQLITE_NOMEM; + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + &pRtree->pNodeBlob); + sqlite3_free(zTab); + } + if( rc ){ + nodeBlobReset(pRtree); + *ppNode = 0; + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); + nodeReference(pParent); } } - rc = sqlite3_reset(pRtree->pReadNode); - if( rc==SQLITE_OK ) rc = rc2; /* If the root node was just loaded, set pRtree->iDepth to the height ** of the r-tree structure. A height of zero means all data is stored on @@ -909,6 +935,7 @@ static void rtreeReference(Rtree *pRtree){ static void rtreeRelease(Rtree *pRtree){ pRtree->nBusy--; if( pRtree->nBusy==0 ){ + nodeBlobReset(pRtree); sqlite3_finalize(pRtree->pReadNode); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); @@ -947,6 +974,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ if( !zCreate ){ rc = SQLITE_NOMEM; }else{ + nodeBlobReset(pRtree); rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); } @@ -3152,6 +3180,27 @@ constraint: return rc; } +/* +** Called when a transaction starts. +** This is a no-op. But the Virtual Table mechanism needs a method +** here or else it will never call the xRollback and xCommit methods, +** and those methods are necessary for clearing the sqlite3_blob object. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + (void)pVtab; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + /* ** The xRename method for rtree module virtual tables. */ @@ -3173,6 +3222,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } + /* ** This function populates the pRtree->nRowEst variable with an estimate ** of the number of rows in the virtual table. If possible, this is based @@ -3232,15 +3282,15 @@ static sqlite3_module rtreeModule = { rtreeColumn, /* xColumn - read data */ rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ - 0, /* xBegin - begin transaction */ + rtreeBeginTransaction, /* xBegin - begin transaction */ 0, /* xSync - sync transaction */ - 0, /* xCommit - commit transaction */ - 0, /* xRollback - rollback transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ }; static int rtreeSqlInit( @@ -3440,7 +3490,7 @@ static int rtreeInit( pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; pRtree->nDim = (u8)((argc-4)/2); - pRtree->nDim2 = argc - 4; + pRtree->nDim2 = pRtree->nDim*2; pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); diff --git a/manifest b/manifest index 4cc36f422d..65dc3b7b4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\san\sexperimental\spatch\sthat\sensures\sthat\sall\scursors\shave\stheir\sposition\nsaved\sprior\sto\sstarting\sa\sROLLBACK\sTO. -D 2017-02-02T00:46:55.000 +C Use\sthe\ssqlite3_blob\sinterface\sfor\sreading\svalues\sfrom\sthe\s%_node\sshadow\ntable\sin\sRTREE.\s\sThis\sis\sa\swork\sin\sprogress.\s\sThere\sare\sstill\ssome\sminor\nproblems. +D 2017-02-02T02:28:45.543 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 70f2488eef5c04dccf15a52cbd4961492124f825 +F ext/rtree/rtree.c 73c4308585c47a7500b9e98617e45a62f8564ddb F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,10 +1552,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c66cf0f0a9ada2ddcb8d61001ef791b86226416 -R 329580c21139f900f64001f6acfc9557 -T *branch * savepoint-rollback -T *sym-savepoint-rollback * -T -sym-trunk * +P 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 +R 79c2c34c9473af6d4e0161547969b398 +T *branch * rtree-sqlite3_blob +T *sym-rtree-sqlite3_blob * +T -sym-savepoint-rollback * U drh -Z 5087946e64391fb6ac59f69d722efdd6 +Z fc394298cc4191cf41d9c752fa550c9c diff --git a/manifest.uuid b/manifest.uuid index 9d5ad1c981..099f46fdff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01d97e5b6502b1811b52a681f445e1aaae6c0ee6 \ No newline at end of file +fc4917d730b29b0bf60fea5e0166728635783e9c \ No newline at end of file From 2033d1c8ca2698a874709b87c04caba559985985 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 14:40:06 +0000 Subject: [PATCH 108/292] Change RTREE so that the sqlite3_blob object is closed whenever the cursor count drops to zero and there is not a pending write transaction. FossilOrigin-Name: 9bb4eafe1a60176ed2e731bb7e3067c0b8a46615 --- ext/rtree/rtree.c | 26 +++++++++++++++++++------- ext/rtree/rtreeA.test | 2 +- manifest | 17 +++++++---------- manifest.uuid | 2 +- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index eae9ef819a..92ddcf526e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -119,11 +119,13 @@ struct Rtree { u8 nDim2; /* Twice the number of dimensions */ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - int nBusy; /* Current number of users of this structure */ + u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -622,7 +624,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob ){ + if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ sqlite3_blob_close(pRtree->pNodeBlob); pRtree->pNodeBlob = 0; } @@ -661,7 +663,7 @@ static int nodeAcquire( pRtree->pNodeBlob = pBlob; if( rc ){ nodeBlobReset(pRtree); - if( rc==SQLITE_NOMEM ) return rc; + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; } } if( pRtree->pNodeBlob==0 ){ @@ -674,6 +676,9 @@ static int nodeAcquire( if( rc ){ nodeBlobReset(pRtree); *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB; }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); if( !pNode ){ @@ -935,6 +940,8 @@ static void rtreeReference(Rtree *pRtree){ static void rtreeRelease(Rtree *pRtree){ pRtree->nBusy--; if( pRtree->nBusy==0 ){ + pRtree->inWrTrans = 0; + pRtree->nCursor = 0; nodeBlobReset(pRtree); sqlite3_finalize(pRtree->pReadNode); sqlite3_finalize(pRtree->pWriteNode); @@ -990,6 +997,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ */ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; RtreeCursor *pCsr; pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); @@ -997,6 +1005,7 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCsr, 0, sizeof(RtreeCursor)); pCsr->base.pVtab = pVTab; rc = SQLITE_OK; + pRtree->nCursor++; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -1029,10 +1038,13 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; + assert( pRtree->nCursor>0 ); freeCursorConstraints(pCsr); sqlite3_free(pCsr->aPoint); for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); + pRtree->nCursor--; + nodeBlobReset(pRtree); return SQLITE_OK; } @@ -3182,12 +3194,11 @@ constraint: /* ** Called when a transaction starts. -** This is a no-op. But the Virtual Table mechanism needs a method -** here or else it will never call the xRollback and xCommit methods, -** and those methods are necessary for clearing the sqlite3_blob object. */ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ - (void)pVtab; + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans++; return SQLITE_OK; } @@ -3197,6 +3208,7 @@ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ */ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; nodeBlobReset(pRtree); return SQLITE_OK; } diff --git a/ext/rtree/rtreeA.test b/ext/rtree/rtreeA.test index e377b013c1..84644e9ede 100644 --- a/ext/rtree/rtreeA.test +++ b/ext/rtree/rtreeA.test @@ -109,7 +109,7 @@ do_corruption_tests rtreeA-1.1 { } do_execsql_test rtreeA-1.2.0 { DROP TABLE t1_node } {} -do_corruption_tests rtreeA-1.2 -error "SQL logic error or missing database" { +do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" { 1 "SELECT * FROM t1" 2 "SELECT * FROM t1 WHERE rowid=5" 3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)" diff --git a/manifest b/manifest index 65dc3b7b4d..5b06a60ca8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\ssqlite3_blob\sinterface\sfor\sreading\svalues\sfrom\sthe\s%_node\sshadow\ntable\sin\sRTREE.\s\sThis\sis\sa\swork\sin\sprogress.\s\sThere\sare\sstill\ssome\sminor\nproblems. -D 2017-02-02T02:28:45.543 +C Change\sRTREE\sso\sthat\sthe\ssqlite3_blob\sobject\sis\sclosed\swhenever\sthe\scursor\ncount\sdrops\sto\szero\sand\sthere\sis\snot\sa\spending\swrite\stransaction. +D 2017-02-02T14:40:06.876 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 73c4308585c47a7500b9e98617e45a62f8564ddb +F ext/rtree/rtree.c df33d86df608a16516a95c6d184a5ed9988d2bc9 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -275,7 +275,7 @@ F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test b5eb13849545dfd271a54ff16784cb00d8792aea -F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf +F ext/rtree/rtreeA.test ac8b503931f2f397cc3c3303354e1e085dcadb86 F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d F ext/rtree/rtreeD.test bdfaaf26df8b4eea7364039aca9150bc1e1f8825 @@ -1552,10 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 -R 79c2c34c9473af6d4e0161547969b398 -T *branch * rtree-sqlite3_blob -T *sym-rtree-sqlite3_blob * -T -sym-savepoint-rollback * +P fc4917d730b29b0bf60fea5e0166728635783e9c +R 0b53bd94b2e2abfad755fb806046ae11 U drh -Z fc394298cc4191cf41d9c752fa550c9c +Z 9c59d9d407f6159f87ae8b113cdc33c2 diff --git a/manifest.uuid b/manifest.uuid index 099f46fdff..0b51f62de3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc4917d730b29b0bf60fea5e0166728635783e9c \ No newline at end of file +9bb4eafe1a60176ed2e731bb7e3067c0b8a46615 \ No newline at end of file From 413e207e316ae584dd2218905ab02522cc8e5a33 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 15:35:54 +0000 Subject: [PATCH 109/292] The sqlite3_blob_close() interface can cause recursive invocations of nodeBlobReset() in RTREE. Make sure that does not cause problems. FossilOrigin-Name: 88333441cbf26bfde2acebf2a3f75b5ebbdfb0ae --- ext/rtree/rtree.c | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 92ddcf526e..c5998d2548 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -625,8 +625,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ */ static void nodeBlobReset(Rtree *pRtree){ if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob_close(pRtree->pNodeBlob); + sqlite3_blob *pBlob = pRtree->pNodeBlob; pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } } diff --git a/manifest b/manifest index 5b06a60ca8..f8a355c0e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sRTREE\sso\sthat\sthe\ssqlite3_blob\sobject\sis\sclosed\swhenever\sthe\scursor\ncount\sdrops\sto\szero\sand\sthere\sis\snot\sa\spending\swrite\stransaction. -D 2017-02-02T14:40:06.876 +C The\ssqlite3_blob_close()\sinterface\scan\scause\srecursive\sinvocations\sof\nnodeBlobReset()\sin\sRTREE.\s\sMake\ssure\sthat\sdoes\snot\scause\sproblems. +D 2017-02-02T15:35:54.700 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c df33d86df608a16516a95c6d184a5ed9988d2bc9 +F ext/rtree/rtree.c 93fd417d6f566caa49a548c915a27125a1b405c3 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc4917d730b29b0bf60fea5e0166728635783e9c -R 0b53bd94b2e2abfad755fb806046ae11 +P 9bb4eafe1a60176ed2e731bb7e3067c0b8a46615 +R 6cebe0517919da4320850125e35977ce U drh -Z 9c59d9d407f6159f87ae8b113cdc33c2 +Z 7ae516135e0fc0897102b9fc210a106c diff --git a/manifest.uuid b/manifest.uuid index 0b51f62de3..ee09db23cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bb4eafe1a60176ed2e731bb7e3067c0b8a46615 \ No newline at end of file +88333441cbf26bfde2acebf2a3f75b5ebbdfb0ae \ No newline at end of file From ce655a236773fafcd3ce751b23cd217a96e5467b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 16:08:27 +0000 Subject: [PATCH 110/292] Fix a potential uninitialized (though harmless) variable in RTREE. FossilOrigin-Name: a1c74e09d63aca630d022ed074866433eed6b493 --- ext/rtree/rtree.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index c5998d2548..432c72781e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1118,7 +1118,6 @@ static int rtreeCallbackConstraint( sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ int *peWithin /* OUT: visibility of the cell */ ){ - int i; /* Loop counter */ sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ int nCoord = pInfo->nCoord; /* No. of coordinates */ int rc; /* Callback return code */ @@ -1163,9 +1162,10 @@ static int rtreeCallbackConstraint( } } if( pConstraint->op==RTREE_MATCH ){ + int eWithin = 0; rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, - nCoord, aCoord, &i); - if( i==0 ) *peWithin = NOT_WITHIN; + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; *prScore = RTREE_ZERO; }else{ pInfo->aCoord = aCoord; diff --git a/manifest b/manifest index f8a355c0e7..0fcd596e1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssqlite3_blob_close()\sinterface\scan\scause\srecursive\sinvocations\sof\nnodeBlobReset()\sin\sRTREE.\s\sMake\ssure\sthat\sdoes\snot\scause\sproblems. -D 2017-02-02T15:35:54.700 +C Fix\sa\spotential\suninitialized\s(though\sharmless)\svariable\sin\sRTREE. +D 2017-02-02T16:08:27.041 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 93fd417d6f566caa49a548c915a27125a1b405c3 +F ext/rtree/rtree.c e1b77d0f32241f3c09ab0bc7cf7be5d263902978 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9bb4eafe1a60176ed2e731bb7e3067c0b8a46615 -R 6cebe0517919da4320850125e35977ce +P 88333441cbf26bfde2acebf2a3f75b5ebbdfb0ae +R 6c083917a4ecdc09328902e3d7d74ce0 U drh -Z 7ae516135e0fc0897102b9fc210a106c +Z 3e4bf024e32b2f321d57a48b2d7cd9ed diff --git a/manifest.uuid b/manifest.uuid index ee09db23cc..507bf4c42c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88333441cbf26bfde2acebf2a3f75b5ebbdfb0ae \ No newline at end of file +a1c74e09d63aca630d022ed074866433eed6b493 \ No newline at end of file From 3accc7e1af59b89cc54df381ddf67dc96487103f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Feb 2017 16:30:25 +0000 Subject: [PATCH 111/292] Remove the unused pReadNode prepared statement from each RTREE object. FossilOrigin-Name: e51dc0ec60d45cd57564735b6b2bb254a588533e --- ext/rtree/rtree.c | 24 ++++++++++-------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 432c72781e..9e6f0e082a 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -139,7 +139,6 @@ struct Rtree { sqlite3_blob *pNodeBlob; /* Statements to read/write/delete a record from xxx_node */ - sqlite3_stmt *pReadNode; sqlite3_stmt *pWriteNode; sqlite3_stmt *pDeleteNode; @@ -944,7 +943,6 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; pRtree->nCursor = 0; nodeBlobReset(pRtree); - sqlite3_finalize(pRtree->pReadNode); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -3315,10 +3313,9 @@ static int rtreeSqlInit( ){ int rc = SQLITE_OK; - #define N_STATEMENT 9 + #define N_STATEMENT 8 static const char *azSql[N_STATEMENT] = { - /* Read and write the xxx_node table */ - "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + /* Write the xxx_node table */ "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", @@ -3356,15 +3353,14 @@ static int rtreeSqlInit( } } - appStmt[0] = &pRtree->pReadNode; - appStmt[1] = &pRtree->pWriteNode; - appStmt[2] = &pRtree->pDeleteNode; - appStmt[3] = &pRtree->pReadRowid; - appStmt[4] = &pRtree->pWriteRowid; - appStmt[5] = &pRtree->pDeleteRowid; - appStmt[6] = &pRtree->pReadParent; - appStmt[7] = &pRtree->pWriteParent; - appStmt[8] = &pRtree->pDeleteParent; + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; rc = rtreeQueryStat1(db, pRtree); for(i=0; i Date: Thu, 2 Feb 2017 19:24:05 +0000 Subject: [PATCH 112/292] Fix issues in the sha1 extension seen with MSVC. FossilOrigin-Name: 5a0da77c22ebc7db5e63b1520d30f3ad97b9bb3b --- ext/misc/sha1.c | 42 +++++++++++++++++++++++++----------------- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index dbf15a95ef..e2843bdefa 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -87,12 +87,6 @@ struct SHA1Context { /* * Hash a single 512-bit block. This is the core of the algorithm. */ -#define a qq[0] -#define b qq[1] -#define c qq[2] -#define d qq[3] -#define e qq[4] - void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ unsigned int qq[5]; /* a, b, c, d, e; */ static int one = 1; @@ -100,6 +94,12 @@ void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ memcpy(block, buffer, 64); memcpy(qq,state,5*sizeof(unsigned int)); +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + /* Copy p->state[] to working vars */ /* a = state[0]; @@ -144,6 +144,12 @@ void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ state[2] += c; state[3] += d; state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e } @@ -253,6 +259,7 @@ static void sha1Func( int nByte = sqlite3_value_bytes(argv[0]); char zOut[44]; + assert( argc==1 ); if( eType==SQLITE_NULL ) return; hash_init(&cx); if( eType==SQLITE_BLOB ){ @@ -292,6 +299,7 @@ static void sha1QueryFunc( SHA1Context cx; char zOut[44]; + assert( argc==1 ); if( zSql==0 ) return; hash_init(&cx); while( zSql[0] ){ @@ -343,7 +351,7 @@ static void sha1QueryFunc( case SQLITE_FLOAT: { sqlite3_uint64 u; int j; - unsigned char x[8]; + unsigned char x[9]; double r = sqlite3_column_double(pStmt,i); memcpy(&u, &r, 8); for(j=8; j>=1; j--){ @@ -355,17 +363,17 @@ static void sha1QueryFunc( break; } case SQLITE_TEXT: { - int n = sqlite3_column_bytes(pStmt, i); - const unsigned char *z = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n); - hash_step(&cx, z, n); + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); break; } case SQLITE_BLOB: { - int n = sqlite3_column_bytes(pStmt, i); - const unsigned char *z = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n); - hash_step(&cx, z, n); + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); break; } } @@ -382,8 +390,8 @@ static void sha1QueryFunc( __declspec(dllexport) #endif int sqlite3_sha_init( - sqlite3 *db, - char **pzErrMsg, + sqlite3 *db, + char **pzErrMsg, const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; diff --git a/manifest b/manifest index 6459c8e5ad..9d4630c832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2017-02-01T23:06:17.747 +C Fix\sissues\sin\sthe\ssha1\sextension\sseen\swith\sMSVC. +D 2017-02-02T19:24:05.672 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -222,7 +222,7 @@ F ext/misc/remember.c 8440f8d0b452c5cdefb62b57135ccd1267aa729d F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 -F ext/misc/sha1.c b2e4eb8e26f09701ec15548395baf698f00e5895 +F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1552,8 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aaae74d06f4865818465cfdb440258ae8a5b985a 997f765bc6706769ae15f3e719354473e02bd78b -R 939bce209d57b72db3d58f9894d61374 -T +closed 997f765bc6706769ae15f3e719354473e02bd78b +P 0c66cf0f0a9ada2ddcb8d61001ef791b86226416 +R 531254e92dc453ecbaf40de729caeadd U mistachkin -Z c2a7081e9858115a573bd4695718af54 +Z 7da98715c0c2c155a034ba4275d6d8fe diff --git a/manifest.uuid b/manifest.uuid index 0e34341685..9e12c1be98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c66cf0f0a9ada2ddcb8d61001ef791b86226416 \ No newline at end of file +5a0da77c22ebc7db5e63b1520d30f3ad97b9bb3b \ No newline at end of file From a7466205ca7f5663605b48375ade5f7389d915a5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Feb 2017 14:44:52 +0000 Subject: [PATCH 113/292] Modify the sqlite3SelectDup() routine to avoid recursing on Select.pPrior. FossilOrigin-Name: a7674ead5be986c66f7d61d598adc7e5728bcd30 --- manifest | 18 +++++++++------- manifest.uuid | 2 +- src/expr.c | 60 +++++++++++++++++++++++++++++---------------------- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index a07bc6f332..24ce068e9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sall\scursors\shave\stheir\spositions\ssaved\sprior\sto\srolling\sback\na\ssavepoint. -D 2017-02-02T20:32:28.731 +C Modify\sthe\ssqlite3SelectDup()\sroutine\sto\savoid\srecursing\son\sSelect.pPrior. +D 2017-02-03T14:44:52.406 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -346,7 +346,7 @@ F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c d7ef6fdb70bc18259d7d36b22aa0bc4845c5c5b9 +F src/expr.c d29114e9b709eaeaaa18553a5bbe60a19302aeef F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -1552,8 +1552,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a0da77c22ebc7db5e63b1520d30f3ad97b9bb3b 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 -R f67fbd165c607c8ab8b9fb8cff6e4d0d -T +closed 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 -U drh -Z d16d6898d272ff11e9984476ae1d9ef3 +P 8e03a8e95fada5c24d369672a71f6e02288051da +R c80f7dbbf5f0044354e4b463e716c13e +T *branch * recursive-selectdup +T *sym-recursive-selectdup * +T -sym-trunk * +U dan +Z f43da81961eb4c39992192293f7e5a2c diff --git a/manifest.uuid b/manifest.uuid index 792cc0b3a3..a72c232e1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e03a8e95fada5c24d369672a71f6e02288051da \ No newline at end of file +a7674ead5be986c66f7d61d598adc7e5728bcd30 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 18035bd421..f1e8667b39 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1418,33 +1418,41 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } return pNew; } -Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew, *pPrior; +Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + Select *p; + assert( db!=0 ); - if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); - pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); - pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); - pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); - pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); - pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); - pNew->op = p->op; - pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); - if( pPrior ) pPrior->pNext = pNew; - pNew->pNext = 0; - pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); - pNew->iLimit = 0; - pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->nSelectRow = p->nSelectRow; - pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); - return pNew; + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = withDup(db, p->pWith); + sqlite3SelectSetName(pNew, p->zSelName); + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + + return pRet; } #else Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ From 59a40db3d2b5cbec56aa12cd0eee403a95c70933 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Feb 2017 15:16:25 +0000 Subject: [PATCH 114/292] Improved tests for the carray() table-valued function. FossilOrigin-Name: 83a099f139aba03edac19c890a0019e922032a25 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/tabfunc01.test | 40 ++++++++++++++++++++++++---------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index a07bc6f332..b4fd3e2062 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sall\scursors\shave\stheir\spositions\ssaved\sprior\sto\srolling\sback\na\ssavepoint. -D 2017-02-02T20:32:28.731 +C Improved\stests\sfor\sthe\scarray()\stable-valued\sfunction. +D 2017-02-03T15:16:25.769 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -1155,7 +1155,7 @@ F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 8b2ef53caa37854864c89e1e57e8a10efd4f5e43 +F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1552,8 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a0da77c22ebc7db5e63b1520d30f3ad97b9bb3b 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 -R f67fbd165c607c8ab8b9fb8cff6e4d0d -T +closed 01d97e5b6502b1811b52a681f445e1aaae6c0ee6 +P 8e03a8e95fada5c24d369672a71f6e02288051da +R 408679a2eb27f8712868946a0e8b1f65 U drh -Z d16d6898d272ff11e9984476ae1d9ef3 +Z 773cdc68fe82e971a45f4eb8ed248cd4 diff --git a/manifest.uuid b/manifest.uuid index 792cc0b3a3..14c0befdff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e03a8e95fada5c24d369672a71f6e02288051da \ No newline at end of file +83a099f139aba03edac19c890a0019e922032a25 \ No newline at end of file diff --git a/test/tabfunc01.test b/test/tabfunc01.test index f25d070847..dcaafa420c 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -148,60 +148,68 @@ do_execsql_test tabfunc01-600 { do_test tabfunc01-700 { - set PTR [intarray_addr 5 7 13 17 23] + set PTR1 [intarray_addr 5 7 13 17 23] db eval { - SELECT b FROM t600, carray($PTR,5) WHERE a=value; + SELECT b FROM t600, carray($PTR1,5) WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-701 { db eval { - SELECT b FROM t600 WHERE a IN carray($PTR,5,'int32'); + SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int32'); } } {(005) (007) (013) (017) (023)} do_test tabfunc01-702 { db eval { - SELECT b FROM t600 WHERE a IN carray($PTR,4,'int32'); + SELECT b FROM t600 WHERE a IN carray($PTR1,4,'int32'); } } {(005) (007) (013) (017)} do_catchsql_test tabfunc01-710 { - SELECT b FROM t600 WHERE a IN carray($PTR,5,'int33'); + SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int33'); } {1 {unknown datatype: 'int33'}} do_test tabfunc01-720 { - set PTR [int64array_addr 5 7 13 17 23] + set PTR2 [int64array_addr 5 7 13 17 23] db eval { - SELECT b FROM t600, carray($PTR,5,'int64') WHERE a=value; + SELECT b FROM t600, carray($PTR2,5,'int64') WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-721 { db eval { - SELECT remember(123,$PTR); - SELECT value FROM carray($PTR,5,'int64'); + SELECT remember(123,$PTR2); + SELECT value FROM carray($PTR2,5,'int64'); } } {123 123 7 13 17 23} do_test tabfunc01-722 { - set PTR2 [expr {$PTR+16}] + set PTR3 [expr {$PTR2+16}] db eval { - SELECT remember(987,$PTR2); - SELECT value FROM carray($PTR,5,'int64'); + SELECT remember(987,$PTR3); + SELECT value FROM carray($PTR2,5,'int64'); } } {987 123 7 987 17 23} do_test tabfunc01-730 { - set PTR [doublearray_addr 5.0 7.0 13.0 17.0 23.0] + set PTR4 [doublearray_addr 5.0 7.0 13.0 17.0 23.0] db eval { - SELECT b FROM t600, carray($PTR,5,'double') WHERE a=value; + SELECT b FROM t600, carray($PTR4,5,'double') WHERE a=value; } } {(005) (007) (013) (017) (023)} do_test tabfunc01-740 { - set PTR [textarray_addr 5 7 13 17 23] + set PTR5 [textarray_addr x5 x7 x13 x17 x23] db eval { - SELECT b FROM t600, carray($PTR,5,'char*') WHERE a=value; + SELECT b FROM t600, carray($PTR5,5,'char*') WHERE a=trim(value,'x'); } } {(005) (007) (013) (017) (023)} +do_test tabfunc01-750 { + db eval { + SELECT aa.value, bb.value, '|' + FROM carray($PTR4,5,'double') AS aa + JOIN carray($PTR5,5,'char*') AS bb ON aa.rowid=bb.rowid; + } +} {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |} +# Free up memory allocations intarray_addr int64array_addr doublearray_addr From 067cd837fbe3014b8876dc4f752babcef0ce5506 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Feb 2017 19:16:39 +0000 Subject: [PATCH 115/292] Avoid a performance problem when very large "VALUES(..), (..), (..)" terms are used in queries. FossilOrigin-Name: f5306ad6816cc377036685cdae227e762885229c --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/select.c | 28 ++++++++++++++++++---------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 24ce068e9c..9a2d78f7e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\ssqlite3SelectDup()\sroutine\sto\savoid\srecursing\son\sSelect.pPrior. -D 2017-02-03T14:44:52.406 +C Avoid\sa\sperformance\sproblem\swhen\svery\slarge\s"VALUES(..),\s(..),\s(..)"\sterms\sare\nused\sin\squeries. +D 2017-02-03T19:16:39.919 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -393,7 +393,7 @@ F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 3856db523b942062bca8722ba03b61c324ff94d6 +F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1552,10 +1552,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8e03a8e95fada5c24d369672a71f6e02288051da -R c80f7dbbf5f0044354e4b463e716c13e -T *branch * recursive-selectdup -T *sym-recursive-selectdup * -T -sym-trunk * +P a7674ead5be986c66f7d61d598adc7e5728bcd30 +R 1275850dbdc882ce3db2a9a335b19414 U dan -Z f43da81961eb4c39992192293f7e5a2c +Z aab63e41186ee170ee1c431b74360517 diff --git a/manifest.uuid b/manifest.uuid index a72c232e1b..c0409900d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7674ead5be986c66f7d61d598adc7e5728bcd30 \ No newline at end of file +f5306ad6816cc377036685cdae227e762885229c \ No newline at end of file diff --git a/src/select.c b/src/select.c index ed6221309f..d817ebd074 100644 --- a/src/select.c +++ b/src/select.c @@ -4186,7 +4186,15 @@ static int withExpand( pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; - sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + if( bMayRecursive ){ + Select *pPrior = pSel->pPrior; + assert( pPrior->pWith==0 ); + pPrior->pWith = pSel->pWith; + sqlite3WalkSelect(pWalker, pPrior); + pPrior->pWith = 0; + }else{ + sqlite3WalkSelect(pWalker, pSel); + } pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); @@ -4230,10 +4238,12 @@ static int withExpand( */ static void selectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - With *pWith = findRightmost(p)->pWith; - if( pWith!=0 ){ - assert( pParse->pWith==pWith ); - pParse->pWith = pWith->pOuter; + if( pParse->pWith && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } } } #else @@ -4283,8 +4293,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - if( pWalker->xSelectCallback2==selectPopWith ){ - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( p->pWith ){ + sqlite3WithPush(pParse, p->pWith, 0); } /* Make sure cursor numbers have been assigned to all entries in @@ -4571,9 +4581,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - if( (pSelect->selFlags & SF_MultiValue)==0 ){ - w.xSelectCallback2 = selectPopWith; - } + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } From 010e312f8fd5f0e73ed83740c3778f3a4e354b35 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Feb 2017 13:12:12 +0000 Subject: [PATCH 116/292] Close sqlite3_blob objects on xSync rather than waiting until xCommit. FossilOrigin-Name: 95ee745fceb4a48c683f34c404c380fe5e7d684a --- ext/rtree/rtree.c | 2 +- manifest | 15 +++++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 9e6f0e082a..c46bce7d37 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3294,7 +3294,7 @@ static sqlite3_module rtreeModule = { rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ rtreeBeginTransaction, /* xBegin - begin transaction */ - 0, /* xSync - sync transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ rtreeEndTransaction, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ diff --git a/manifest b/manifest index ea1bf5a3bf..e59dba6295 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\spReadNode\sprepared\sstatement\sfrom\seach\sRTREE\sobject. -D 2017-02-02T16:30:25.555 +C Close\ssqlite3_blob\sobjects\son\sxSync\srather\sthan\swaiting\suntil\sxCommit. +D 2017-02-04T13:12:12.865 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c be9d44f5707c5a73887d3ac3ad14a355e9b92b58 +F ext/rtree/rtree.c ed39c157eab89ce014675f07601f2813c9f54e9c F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1552,7 +1552,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1c74e09d63aca630d022ed074866433eed6b493 -R b4b0a5df6a2ab40920c13363cd1d25e9 +P e51dc0ec60d45cd57564735b6b2bb254a588533e +R a16debe67e7399020e94d725cd2d4af7 +T *branch * rtree-blob-agressive-release +T *sym-rtree-blob-agressive-release * +T -sym-rtree-sqlite3_blob * U drh -Z 5406bbb7ccb82072da8c73ecd9c87865 +Z a61d538b0d9c1a9c54d88c1f71c254d7 diff --git a/manifest.uuid b/manifest.uuid index dda662b8b9..be149e116e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e51dc0ec60d45cd57564735b6b2bb254a588533e \ No newline at end of file +95ee745fceb4a48c683f34c404c380fe5e7d684a \ No newline at end of file From ff677b20fc34a4fd7b4c0ab3c499ec355d631ad2 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Feb 2017 17:33:30 +0000 Subject: [PATCH 117/292] Allow sqlite3session_apply() to apply changesets to tables that have been extended using ALTER TABLE ADD COLUMN. FossilOrigin-Name: b20ff81ff9c8af500ea96e0ba9d34524220a89f1 --- ext/session/session3.test | 5 +- ext/session/session_common.tcl | 13 +- ext/session/sessionat.test | 216 +++++++++++++++++++++++++++++++++ ext/session/sqlite3session.c | 39 ++++-- ext/session/sqlite3session.h | 25 ++-- manifest | 24 ++-- manifest.uuid | 2 +- test/tester.tcl | 31 ++++- 8 files changed, 312 insertions(+), 43 deletions(-) create mode 100644 ext/session/sessionat.test diff --git a/ext/session/session3.test b/ext/session/session3.test index e15407c2eb..ba316348ef 100644 --- a/ext/session/session3.test +++ b/ext/session/session3.test @@ -63,7 +63,10 @@ do_test 1.2.1 { INSERT INTO t1 VALUES(7, 8); } set ::log -} {SQLITE_SCHEMA {sqlite3changeset_apply(): table t1 has 3 columns, expected 2}} +} {} +do_test 1.2.2 { + db2 eval { SELECT * FROM t1 } +} {5 6 {} 7 8 {}} do_test 1.3.0 { execsql { diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl index 06b05509b1..d4804d924f 100644 --- a/ext/session/session_common.tcl +++ b/ext/session/session_common.tcl @@ -33,16 +33,11 @@ proc do_changeset_invert_test {tn session res} { proc do_conflict_test {tn args} { - proc xConflict {args} { - lappend ::xConflict $args - return "" - } - proc bgerror {args} { set ::background_error $args } - set O(-tables) [list] set O(-sql) [list] set O(-conflicts) [list] + set O(-policy) "OMIT" array set V $args foreach key [array names V] { @@ -50,6 +45,12 @@ proc do_conflict_test {tn args} { } array set O $args + proc xConflict {args} [subst -nocommands { + lappend ::xConflict [set args] + return $O(-policy) + }] + proc bgerror {args} { set ::background_error $args } + sqlite3session S db main foreach t $O(-tables) { S attach $t } execsql $O(-sql) diff --git a/ext/session/sessionat.test b/ext/session/sessionat.test new file mode 100644 index 0000000000..a96cbcae42 --- /dev/null +++ b/ext/session/sessionat.test @@ -0,0 +1,216 @@ +# 2017 February 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the sessions module. Specifically, that a changeset can +# be applied after ALTER TABLE ADD COLUMN has been used to add +# columns to tables. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionat + +db close +sqlite3_shutdown +test_sqlite3_log log +proc log {code msg} { lappend ::log $code $msg } + +proc reset_test {} { + catch { db close } + catch { db2 close } + forcedelete test.db test.db2 + sqlite3 db test.db + sqlite3 db2 test.db2 +} + + +# Run all tests in this file twice. Once with "WITHOUT ROWID", and once +# with regular rowid tables. +# +# ?.1.*: Test that PK inconsistencies are detected if one or more of the PK +# columns are not present in the changeset. +# +# ?.2.*: Test that it is not possible to apply a changeset with N columns +# to a db with fewer than N columns. +# +# ?.3.*: Test some INSERT, UPDATE and DELETE operations that do not +# require conflict handling. +# +# ?.4.*: Test some INSERT, UPDATE and DELETE operations that do require +# conflict handling. +# +# ?.5.*: Test that attempting to concat two changesets with different +# numbers of columns for the same table is an error. +# +foreach {tn trailing} { + sessionat-ipk "" + sessionat-wor " WITHOUT ROWID " +} { +eval [string map [list %WR% $trailing] { + reset_test + + #----------------------------------------------------------------------- + do_execsql_test $tn.1.0 { + CREATE TABLE t1(a, b, PRIMARY KEY(a)) %WR%; + } + do_execsql_test -db db2 $tn.1.1 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) %WR%; + } + do_test $tn.1.2 { + set ::log {} + do_then_apply_sql { INSERT INTO t1 VALUES('one', 'two') } + set ::log + } [list \ + SQLITE_SCHEMA {sqlite3changeset_apply(): primary key mismatch for table t1} + ] + do_execsql_test $tn.1.3 { SELECT * FROM t1 } {one two} + do_execsql_test -db db2 $tn.1.4 { SELECT * FROM t1 } {} + + #----------------------------------------------------------------------- + do_execsql_test $tn.2.0 { + CREATE TABLE t2(x, y, z, PRIMARY KEY(x)) %WR%; + } + do_execsql_test -db db2 $tn.2.1 { + CREATE TABLE t2(x, y, PRIMARY KEY(x)) %WR%; + } + do_test $tn.2.2 { + db cache flush + set ::log {} + do_then_apply_sql { INSERT INTO t2 VALUES(1, 2, 3) } + set ::log + } [list SQLITE_SCHEMA \ + {sqlite3changeset_apply(): table t2 has 2 columns, expected 3 or more} + ] + do_execsql_test $tn.2.3 { SELECT * FROM t2 } {1 2 3} + do_execsql_test -db db2 $tn.2.4 { SELECT * FROM t2 } {} + + #----------------------------------------------------------------------- + do_execsql_test $tn.3.0 { + CREATE TABLE t3(a, b, PRIMARY KEY(b)) %WR%; + } + do_execsql_test -db db2 $tn.3.1 { + CREATE TABLE t3(a, b, c DEFAULT 'D', PRIMARY KEY(b)) %WR%; + } + do_test $tn.3.2 { + do_then_apply_sql { + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(3, 4); + INSERT INTO t3 VALUES(5, 6); + }; + db2 eval {SELECT * FROM t3} + } {1 2 D 3 4 D 5 6 D} + do_test $tn.3.3 { + do_then_apply_sql { + UPDATE t3 SET a=45 WHERE b=4; + DELETE FROM t3 WHERE a=5; + }; + db2 eval {SELECT * FROM t3} + } {1 2 D 45 4 D} + + #----------------------------------------------------------------------- + # 4.1: INSERT statements + # 4.2: DELETE statements + # 4.3: UPDATE statements + # + do_execsql_test $tn.4.1.0 { + CREATE TABLE t4(x INTEGER PRIMARY KEY, y) %WR%; + } + do_execsql_test -db db2 $tn.4.1.1 { + CREATE TABLE t4(x INTEGER PRIMARY KEY, y, z) %WR%; + INSERT INTO t4 VALUES(1, 2, 3); + INSERT INTO t4 VALUES(4, 5, 6); + } + do_conflict_test $tn.4.1.2 -tables t4 -sql { + INSERT INTO t4 VALUES(10, 20); + INSERT INTO t4 VALUES(4, 11); + } -conflicts { + {INSERT t4 CONFLICT {i 4 i 11} {i 4 i 5}} + } + do_execsql_test -db db2 $tn.4.1.3 { + SELECT * FROM t4 ORDER BY x + } {1 2 3 4 5 6 10 20 {}} + do_conflict_test $tn.4.1.4 -policy REPLACE -tables t4 -sql { + INSERT INTO t4 VALUES(1, 11); + } -conflicts { + {INSERT t4 CONFLICT {i 1 i 11} {i 1 i 2}} + } + do_execsql_test -db db2 $tn.4.1.5 { + SELECT * FROM t4 ORDER BY x + } {1 11 {} 4 5 6 10 20 {}} + + do_execsql_test $tn.4.2.0 { + DELETE FROM t4; + INSERT INTO t4 VALUES(1, 'A'); + INSERT INTO t4 VALUES(2, 'B'); + INSERT INTO t4 VALUES(3, 'C'); + INSERT INTO t4 VALUES(4, 'D'); + } + do_execsql_test -db db2 $tn.4.2.1 { + DELETE FROM t4; + INSERT INTO t4 VALUES(1, 'A', 'a'); + INSERT INTO t4 VALUES(3, 'C', 'c'); + INSERT INTO t4 VALUES(4, 'E', 'd'); + } + do_conflict_test $tn.4.2.2 -tables t4 -sql { + DELETE FROM t4 WHERE x=2; + DELETE FROM t4 WHERE x=4; + } -conflicts { + {DELETE t4 NOTFOUND {i 2 t B}} + {DELETE t4 DATA {i 4 t D} {i 4 t E}} + } + + do_execsql_test $tn.4.3.0 { + CREATE TABLE t5(a, b, c PRIMARY KEY) %WR%; + INSERT INTO t5 VALUES(1,1,1), (2,2,2), (3,3,3), (4,4,4); + } + do_execsql_test -db db2 $tn.4.3.1 { + CREATE TABLE t5(a, b, c PRIMARY KEY, d CHECK(b!=10)) %WR%; + INSERT INTO t5 VALUES (2,2,2,2), (3,8,3,3), (4,4,4,4); + } + do_conflict_test $tn.4.3.2 -tables t5 -sql { + UPDATE t5 SET a=4 WHERE c=1; + UPDATE t5 SET b=9 WHERE c=3; + UPDATE t5 SET b=10 WHERE c=2; + } -conflicts { + {UPDATE t5 NOTFOUND {i 1 {} {} i 1} {i 4 {} {} {} {}}} + {UPDATE t5 DATA {{} {} i 3 i 3} {{} {} i 9 {} {}} {i 3 i 8 i 3}} + {UPDATE t5 CONSTRAINT {{} {} i 2 i 2} {{} {} i 10 {} {}}} + } + + #----------------------------------------------------------------------- + do_execsql_test $tn.5.0 { + CREATE TABLE t6(a, b, c, PRIMARY KEY(a, b)) %WR%; + } + do_execsql_test -db db2 $tn.5.1 { + CREATE TABLE t6(a, b, c, d, e, PRIMARY KEY(a, b)) %WR%; + } + do_test $tn.5.2 { + set c1 [sql_exec_changeset db { + INSERT INTO t6 VALUES(1, 1, 1); + INSERT INTO t6 VALUES(2, 2, 2); + }] + set c2 [sql_exec_changeset db2 { + INSERT INTO t6 VALUES(3, 3, 3, 3, 3); + INSERT INTO t6 VALUES(4, 4, 4, 4, 4); + }] + list [catch { sqlite3changeset_concat $c1 $c2} msg] $msg + } {1 SQLITE_SCHEMA} + +}] +} + + +finish_test diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 50793df0a0..8dde8501a3 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -3028,7 +3028,7 @@ int sqlite3changeset_conflict( if( !pIter->pConflict ){ return SQLITE_MISUSE; } - if( iVal<0 || iVal>=sqlite3_column_count(pIter->pConflict) ){ + if( iVal<0 || iVal>=pIter->nCol ){ return SQLITE_RANGE; } *ppValue = sqlite3_column_value(pIter->pConflict, iVal); @@ -3495,7 +3495,13 @@ static int sessionInsertRow( sessionAppendStr(&buf, "INSERT INTO main.", &rc); sessionAppendIdent(&buf, zTab, &rc); - sessionAppendStr(&buf, " VALUES(?", &rc); + sessionAppendStr(&buf, "(", &rc); + for(i=0; inCol; i++){ + if( i!=0 ) sessionAppendStr(&buf, ", ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + } + + sessionAppendStr(&buf, ") VALUES(?", &rc); for(i=1; inCol; i++){ sessionAppendStr(&buf, ", ?", &rc); } @@ -4041,11 +4047,17 @@ static int sessionChangesetApply( nTab = (int)strlen(zTab); sApply.azCol = (const char **)zTab; }else{ + int nMinCol = 0; + int i; + sqlite3changeset_pk(pIter, &abPK, 0); rc = sessionTableInfo( db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK ); if( rc!=SQLITE_OK ) break; + for(i=0; i For each row (primary key) that exists in both tables, but features -** different in each, an UPDATE record is added to the session. +** different non-PK values in each, an UPDATE record is added to the +** session. ** ** ** To clarify, if this function is called and then a changeset constructed @@ -904,7 +905,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); **
      **
    • The table has the same name as the name recorded in the ** changeset, and -**
    • The table has the same number of columns as recorded in the +**
    • The table has at least as many columns as recorded in the ** changeset, and **
    • The table has primary key columns in the same position as ** recorded in the changeset. @@ -949,7 +950,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** If a row with matching primary key values is found, but one or more of ** the non-primary key fields contains a value different from the original ** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. ** ** If no row with matching primary key values is found in the database, ** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] @@ -964,7 +969,9 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** **
      INSERT Changes
      ** For each INSERT change, an attempt is made to insert the new row into -** the database. +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. ** ** If the attempt to insert the row fails because the database already ** contains a row with the same primary key values, the conflict handler @@ -982,13 +989,13 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** For each UPDATE change, this function checks if the target database ** contains a row with the same primary key value (or values) as the ** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in -** the changeset the row is updated within the target database. +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. ** ** If a row with matching primary key values is found, but one or more of -** the non-primary key fields contains a value different from an original -** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since ** UPDATE changes only contain values for non-primary key fields that are ** to be modified, only those fields need to match the original values to ** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. diff --git a/manifest b/manifest index 1e02448111..0c44921e62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sRTREE,\suse\san\ssqlite3_blob\sobject\srather\sthan\san\ssqlite3_stmt\sobject\nfor\sreading\scontent\sout\sof\sthe\s%_node\sshadow\stable. -D 2017-02-04T14:24:05.401 +C Allow\ssqlite3session_apply()\sto\sapply\schangesets\sto\stables\sthat\shave\sbeen\nextended\susing\sALTER\sTABLE\sADD\sCOLUMN. +D 2017-02-04T17:33:30.792 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -290,7 +290,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test c8a50e0e8581dc1a00e832aa59bb61f180404d44 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 -F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 +F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 @@ -303,13 +303,14 @@ F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2 F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 -F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4 +F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73 F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723 +F ext/session/sessionat.test b25d61d663ebc795506bf74079dc4ba0092fad25 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc -F ext/session/sqlite3session.c c61a43396368ec00dc127f7bc647e9bd6a4ee5fb -F ext/session/sqlite3session.h 9345166bd8f80562145586cf817f707de5ecada2 +F ext/session/sqlite3session.c 13642d9c754cc18f17e141f82860d269e2adf920 +F ext/session/sqlite3session.h d4db650adfcc7a4360e9f12a09c2d117b1db6b53 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 @@ -1167,7 +1168,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 2a49c1aff731f380ea640106377e19611a1443ae +F test/tester.tcl 67835ac17e90055f24a9cf52e5c5bce0dd511c74 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1552,8 +1553,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5706d4708a30eb54da0ecbb6eb02f54746c390d9 95ee745fceb4a48c683f34c404c380fe5e7d684a -R ad6ed02dcc60e4bca51c672c64ac9b20 -T +closed 95ee745fceb4a48c683f34c404c380fe5e7d684a -U drh -Z c59d505acee8b5f741c72c39e5c6cdac +P 97ccf3e4de11ffea46993cb7fb7ab559b9810705 +R cad52333a5be0dfe0e9c0b2fb42ed068 +U dan +Z 4713f44978189d862833bfda87ecea6f diff --git a/manifest.uuid b/manifest.uuid index 99286a49a0..ffbedec871 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97ccf3e4de11ffea46993cb7fb7ab559b9810705 \ No newline at end of file +b20ff81ff9c8af500ea96e0ba9d34524220a89f1 \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 8cc501a182..1da89fec26 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -919,10 +919,37 @@ proc normalize_list {L} { set L2 } -proc do_execsql_test {testname sql {result {}}} { +# Either: +# +# do_execsql_test TESTNAME SQL ?RES? +# do_execsql_test -db DB TESTNAME SQL ?RES? +# +proc do_execsql_test {args} { + set db db + if {[lindex $args 0]=="-db"} { + set db [lindex $args 1] + set args [lrange $args 2 end] + } + + if {[llength $args]==2} { + foreach {testname sql} $args {} + set result "" + } elseif {[llength $args]==3} { + foreach {testname sql result} $args {} + } else { + error [string trim { + wrong # args: should be "do_execsql_test ?-db DB? testname sql ?result?" + }] + } + fix_testname testname - uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] + + uplevel do_test \ + [list $testname] \ + [list "execsql {$sql} $db"] \ + [list [list {*}$result]] } + proc do_catchsql_test {testname sql result} { fix_testname testname uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] From 238390c3b4f9e1ac568e268d716bfd5497d38e69 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Feb 2017 20:15:51 +0000 Subject: [PATCH 118/292] Simplification to the error handling to extension loading in sqlite3_open(). FossilOrigin-Name: ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 34 +++++++++++++++++----------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 0c44921e62..3a187ee6a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\ssqlite3session_apply()\sto\sapply\schangesets\sto\stables\sthat\shave\sbeen\nextended\susing\sALTER\sTABLE\sADD\sCOLUMN. -D 2017-02-04T17:33:30.792 +C Simplification\sto\sthe\serror\shandling\sto\sextension\sloading\sin\s\nsqlite3_open(). +D 2017-02-04T20:15:51.171 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -359,7 +359,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 -F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 +F src/main.c eb4f5034ede07e1777e3e31fb2059a0046f499af F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -1553,7 +1553,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 97ccf3e4de11ffea46993cb7fb7ab559b9810705 -R cad52333a5be0dfe0e9c0b2fb42ed068 -U dan -Z 4713f44978189d862833bfda87ecea6f +P b20ff81ff9c8af500ea96e0ba9d34524220a89f1 +R 001e21c81e4051080c5d23cc6f975774 +U drh +Z 5c4d22afe8177f7d6c2b43d525198743 diff --git a/manifest.uuid b/manifest.uuid index ffbedec871..77acfcf115 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b20ff81ff9c8af500ea96e0ba9d34524220a89f1 \ No newline at end of file +ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 9aad8fdd4c..e771c8eb04 100644 --- a/src/main.c +++ b/src/main.c @@ -2989,13 +2989,13 @@ static int openDatabase( if( rc==SQLITE_OK ){ sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } } + testcase( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ); + testcase( rc!=SQLITE_OK && !db->mallocFailed ); + #ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ + if( !db->mallocFailed && rc==SQLITE_OK ){ extern int sqlite3Fts1Init(sqlite3*); rc = sqlite3Fts1Init(db); } @@ -3038,23 +3038,23 @@ static int openDatabase( } #endif - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ + if( rc==SQLITE_OK ){ + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ #ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); #endif - if( rc ) sqlite3Error(db, rc); + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); - - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + } opendb_out: if( db ){ From 9630f3ba085cf4d3a4b9b198153470cf2fd8fea8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Feb 2017 01:19:07 +0000 Subject: [PATCH 119/292] Remove incorrect testcase() macros from R-Tree. FossilOrigin-Name: 853a58a75ecbd01ebbf7e07fbd8c4fc9bef54a99 --- ext/rtree/rtree.c | 5 ----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 5ea7824e2a..6399d225c2 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -456,21 +456,17 @@ static void readCoord(u8 *p, RtreeCoord *pCoord){ static i64 readInt64(u8 *p){ #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u64 x; - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return (i64)_byteswap_uint64(x); #elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) u64 x; - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return (i64)__builtin_bswap64(x); #elif SQLITE_BYTEORDER==4321 i64 x; - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ memcpy(&x, p, 8); return x; #else - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* not always 8-byte aligned */ return ( (((i64)p[0]) << 56) + (((i64)p[1]) << 48) + @@ -518,7 +514,6 @@ static int writeCoord(u8 *p, RtreeCoord *pCoord){ return 4; } static int writeInt64(u8 *p, i64 i){ - testcase( ((((char*)p) - (char*)0)&7)!=0 ); /* Not always 8-byte aligned */ #if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) i = (i64)__builtin_bswap64((u64)i); memcpy(p, &i, 8); diff --git a/manifest b/manifest index 3a187ee6a7..fb8fafb97b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\serror\shandling\sto\sextension\sloading\sin\s\nsqlite3_open(). -D 2017-02-04T20:15:51.171 +C Remove\sincorrect\stestcase()\smacros\sfrom\sR-Tree. +D 2017-02-06T01:19:07.230 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 0c229d23ce7b9bc69365281689065f3cbbe4561e +F ext/rtree/rtree.c d40feeeee43736bb96fce8a73385ca721c207d4a F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1553,7 +1553,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b20ff81ff9c8af500ea96e0ba9d34524220a89f1 -R 001e21c81e4051080c5d23cc6f975774 +P ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 +R b0af6ea2fd6b4dd25607ca94e5b23382 U drh -Z 5c4d22afe8177f7d6c2b43d525198743 +Z 0dd488b4639ebd89796370a35153554a diff --git a/manifest.uuid b/manifest.uuid index 77acfcf115..7fef035644 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 \ No newline at end of file +853a58a75ecbd01ebbf7e07fbd8c4fc9bef54a99 \ No newline at end of file From 809955b504cbb41daa14152647b990cb36b0965b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Feb 2017 07:37:50 +0000 Subject: [PATCH 120/292] Fix a bug in kvtest causing "init --variance 0" runs to generate single byte blob values only. FossilOrigin-Name: 6b0276f968d3c6430076c6e540907cf840d0f398 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/kvtest.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index fb8fafb97b..bd1b152de9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sincorrect\stestcase()\smacros\sfrom\sR-Tree. -D 2017-02-06T01:19:07.230 +C Fix\sa\sbug\sin\skvtest\scausing\s"init\s--variance\s0"\sruns\sto\sgenerate\ssingle\sbyte\nblob\svalues\sonly. +D 2017-02-06T07:37:50.092 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -903,7 +903,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 7a3c38ee56b9cc45dc5a5edc50fd9bc9425659a9 +F test/kvtest.c 156281b4ed688e9aabf1fbcdc5dec0487b267026 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1553,7 +1553,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 -R b0af6ea2fd6b4dd25607ca94e5b23382 -U drh -Z 0dd488b4639ebd89796370a35153554a +P 853a58a75ecbd01ebbf7e07fbd8c4fc9bef54a99 +R 921673f8e8c6498440ee3e75a7e902ee +U dan +Z ce0aba82a1b0f2c7c9c23b4307d78840 diff --git a/manifest.uuid b/manifest.uuid index 7fef035644..91838993f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -853a58a75ecbd01ebbf7e07fbd8c4fc9bef54a99 \ No newline at end of file +6b0276f968d3c6430076c6e540907cf840d0f398 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 2a108aa470..77ee02a4ff 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -310,7 +310,7 @@ static int initMain(int argc, char **argv){ "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n" "COMMIT;\n", - pgsz, nCount, sz, iVariance + pgsz, nCount, sz, iVariance+1 ); rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); if( rc ) fatalError("database create failed: %s", zErrMsg); From 943aa77a8a68d7e6341d1189537d05e05f37afc0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Feb 2017 15:27:44 +0000 Subject: [PATCH 121/292] Add the "max-limits" utility program to the tools/ subdirectory. FossilOrigin-Name: c54173b6c1acbb41ed9c323099e94b8e02c6c895 --- manifest | 13 +++++++------ manifest.uuid | 2 +- tool/max-limits.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 tool/max-limits.c diff --git a/manifest b/manifest index bd1b152de9..ba5d6d8cf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\skvtest\scausing\s"init\s--variance\s0"\sruns\sto\sgenerate\ssingle\sbyte\nblob\svalues\sonly. -D 2017-02-06T07:37:50.092 +C Add\sthe\s"max-limits"\sutility\sprogram\sto\sthe\stools/\ssubdirectory. +D 2017-02-06T15:27:44.567 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -1485,6 +1485,7 @@ F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca +F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 F tool/mkkeywordhash.c f7f3b342211ac6a14258b9726d5b97cf4f548f22 F tool/mkmsvcmin.tcl 2f12f7fa8858bbe61cf81820a2da96c79ed1ca8d @@ -1553,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 853a58a75ecbd01ebbf7e07fbd8c4fc9bef54a99 -R 921673f8e8c6498440ee3e75a7e902ee -U dan -Z ce0aba82a1b0f2c7c9c23b4307d78840 +P 6b0276f968d3c6430076c6e540907cf840d0f398 +R f80944376543a94e77dae99832d50174 +U drh +Z ae78f6b03eb9477204e774bb8b71a31e diff --git a/manifest.uuid b/manifest.uuid index 91838993f7..c191fc2bb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b0276f968d3c6430076c6e540907cf840d0f398 \ No newline at end of file +c54173b6c1acbb41ed9c323099e94b8e02c6c895 \ No newline at end of file diff --git a/tool/max-limits.c b/tool/max-limits.c new file mode 100644 index 0000000000..d019974426 --- /dev/null +++ b/tool/max-limits.c @@ -0,0 +1,41 @@ +/* +** Link this program against an SQLite library of unknown provenance in order +** to display the compile-time maximum values for various settings. +*/ +#include "sqlite3.h" +#include + +static const struct { + int eCode; + char *zName; +} aLimit[] = { + { SQLITE_LIMIT_LENGTH, "SQLITE_MAX_LENGTH" }, + { SQLITE_LIMIT_SQL_LENGTH, "SQLITE_MAX_SQL_LENGTH" }, + { SQLITE_LIMIT_COLUMN, "SQLITE_MAX_COLUMN" }, + { SQLITE_LIMIT_EXPR_DEPTH, "SQLITE_MAX_EXPR_DEPTH" }, + { SQLITE_LIMIT_COMPOUND_SELECT, "SQLITE_MAX_COMPOUND_SELECT" }, + { SQLITE_LIMIT_VDBE_OP, "SQLITE_MAX_VDBE_OP" }, + { SQLITE_LIMIT_FUNCTION_ARG, "SQLITE_MAX_FUNCTION_ARG" }, + { SQLITE_LIMIT_ATTACHED, "SQLITE_MAX_ATTACHED" }, + { SQLITE_LIMIT_LIKE_PATTERN_LENGTH, "SQLITE_MAX_LIKE_PATTERN_LENGTH" }, + { SQLITE_LIMIT_VARIABLE_NUMBER, "SQLITE_MAX_VARIABLE_NUMBER" }, + { SQLITE_LIMIT_TRIGGER_DEPTH, "SQLITE_MAX_TRIGGER_DEPTH" }, + { SQLITE_LIMIT_WORKER_THREADS, "SQLITE_MAX_WORKER_THREADS" }, +}; + +static int maxLimit(sqlite3 *db, int eCode){ + int iOrig = sqlite3_limit(db, eCode, 0x7fffffff); + return sqlite3_limit(db, eCode, iOrig); +} + +int main(int argc, char **argv){ + sqlite3 *db; + int j, rc; + rc = sqlite3_open(":memory:", &db); + if( rc==SQLITE_OK ){ + for(j=0; j Date: Mon, 6 Feb 2017 16:52:32 +0000 Subject: [PATCH 122/292] Remove a small amount of unnecessary code from R-Tree. FossilOrigin-Name: e5aea894267addb4dc9b21de24a20417b0be508a --- ext/rtree/rtree.c | 18 ++---------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 6399d225c2..4d77be827f 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1611,7 +1611,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - if( rc ) return rc; nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -1806,19 +1805,6 @@ static int rtreeFilter( return rc; } -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} - /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -1898,7 +1884,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** a single row. */ pIdxInfo->estimatedCost = 30.0; - setEstimatedRows(pIdxInfo, 1); + pIdxInfo->estimatedRows = 1; return SQLITE_OK; } @@ -1930,7 +1916,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; - setEstimatedRows(pIdxInfo, nRow); + pIdxInfo->estimatedRows = nRow; return rc; } diff --git a/manifest b/manifest index ba5d6d8cf1..b224942f12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"max-limits"\sutility\sprogram\sto\sthe\stools/\ssubdirectory. -D 2017-02-06T15:27:44.567 +C Remove\sa\ssmall\samount\sof\sunnecessary\scode\sfrom\sR-Tree. +D 2017-02-06T16:52:32.895 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d40feeeee43736bb96fce8a73385ca721c207d4a +F ext/rtree/rtree.c 2f7785eec412bcf289d5088ed74abe9855e8a5d5 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b0276f968d3c6430076c6e540907cf840d0f398 -R f80944376543a94e77dae99832d50174 +P c54173b6c1acbb41ed9c323099e94b8e02c6c895 +R 826a4ad10a5c86df22518befe7373210 U drh -Z ae78f6b03eb9477204e774bb8b71a31e +Z 369aa1878a60b61dc52ce9554d8934f7 diff --git a/manifest.uuid b/manifest.uuid index c191fc2bb5..4b0d774976 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c54173b6c1acbb41ed9c323099e94b8e02c6c895 \ No newline at end of file +e5aea894267addb4dc9b21de24a20417b0be508a \ No newline at end of file From b18bf843f7c2574059e93d41a087485b38e057c4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 00:55:47 +0000 Subject: [PATCH 123/292] Fix a minor typo in a comment in R-Tree. No changes to code. FossilOrigin-Name: f77ee9e941f22b0b2e88871df4466fdde9cde131 --- ext/rtree/rtree.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 4d77be827f..0241a6279a 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1304,7 +1304,7 @@ static int rtreeSearchPointCompare( } /* -** Interchange to search points in a cursor. +** Interchange two search points in a cursor. */ static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ RtreeSearchPoint t = p->aPoint[i]; diff --git a/manifest b/manifest index b224942f12..6545db5863 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssmall\samount\sof\sunnecessary\scode\sfrom\sR-Tree. -D 2017-02-06T16:52:32.895 +C Fix\sa\sminor\stypo\sin\sa\scomment\sin\sR-Tree.\s\sNo\schanges\sto\scode. +D 2017-02-07T00:55:47.045 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 2f7785eec412bcf289d5088ed74abe9855e8a5d5 +F ext/rtree/rtree.c 76311736628e87fd780dfbbbf5813bf1f9bf2329 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c54173b6c1acbb41ed9c323099e94b8e02c6c895 -R 826a4ad10a5c86df22518befe7373210 +P e5aea894267addb4dc9b21de24a20417b0be508a +R d0b5dcf12941f0f1570d0961d3cc15da U drh -Z 369aa1878a60b61dc52ce9554d8934f7 +Z 7c46a9478c1e693d3d76de1002d9aeea diff --git a/manifest.uuid b/manifest.uuid index 4b0d774976..b66e0e2d13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5aea894267addb4dc9b21de24a20417b0be508a \ No newline at end of file +f77ee9e941f22b0b2e88871df4466fdde9cde131 \ No newline at end of file From 76f63789f2815873d05bdd039c0c49ce5ffb070b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 03:44:42 +0000 Subject: [PATCH 124/292] Back out [ec8ff892ac] which is causing missed OOM errors on some tests. FossilOrigin-Name: 0611770d6a2fcb9fa0bedee19df9916820ae8f3b --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/main.c | 34 +++++++++++++++++----------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 6545db5863..7dcfa25be5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sminor\stypo\sin\sa\scomment\sin\sR-Tree.\s\sNo\schanges\sto\scode. -D 2017-02-07T00:55:47.045 +C Back\sout\s[ec8ff892ac]\swhich\sis\scausing\smissed\sOOM\serrors\son\ssome\stests. +D 2017-02-07T03:44:42.867 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -359,7 +359,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 -F src/main.c eb4f5034ede07e1777e3e31fb2059a0046f499af +F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -1554,7 +1554,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e5aea894267addb4dc9b21de24a20417b0be508a -R d0b5dcf12941f0f1570d0961d3cc15da +P f77ee9e941f22b0b2e88871df4466fdde9cde131 +Q -ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 +R ebea043e9962974403d836188c02b08f U drh -Z 7c46a9478c1e693d3d76de1002d9aeea +Z a87f51011f00704857e3612bb8043da6 diff --git a/manifest.uuid b/manifest.uuid index b66e0e2d13..1b41d3e917 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f77ee9e941f22b0b2e88871df4466fdde9cde131 \ No newline at end of file +0611770d6a2fcb9fa0bedee19df9916820ae8f3b \ No newline at end of file diff --git a/src/main.c b/src/main.c index e771c8eb04..9aad8fdd4c 100644 --- a/src/main.c +++ b/src/main.c @@ -2989,13 +2989,13 @@ static int openDatabase( if( rc==SQLITE_OK ){ sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } } - testcase( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ); - testcase( rc!=SQLITE_OK && !db->mallocFailed ); - #ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed && rc==SQLITE_OK ){ + if( !db->mallocFailed ){ extern int sqlite3Fts1Init(sqlite3*); rc = sqlite3Fts1Init(db); } @@ -3038,23 +3038,23 @@ static int openDatabase( } #endif - if( rc==SQLITE_OK ){ - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ #ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); #endif - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); + if( rc ) sqlite3Error(db, rc); - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); - } + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: if( db ){ From edd9bcb372774046fe940329ce96bdd9c7ca30f4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 12:58:38 +0000 Subject: [PATCH 125/292] Make the cellMargin() routine of R-Tree slightly smaller and faster while also fixing a harmless compiler warning. FossilOrigin-Name: 07fe6228208684d579c4f6c334c90eb6262a9233 --- ext/rtree/rtree.c | 10 +++++----- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 0241a6279a..e898e34b79 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1955,12 +1955,12 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ ** of the objects size in each dimension. */ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin; - int ii; - margin = DCOORD(p->aCoord[1]) - DCOORD(p->aCoord[0]); - for(ii=2; iinDim2; ii+=2){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); - } + ii -= 2; + }while( ii>=0 ); return margin; } diff --git a/manifest b/manifest index 7dcfa25be5..255e0df908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\s[ec8ff892ac]\swhich\sis\scausing\smissed\sOOM\serrors\son\ssome\stests. -D 2017-02-07T03:44:42.867 +C Make\sthe\scellMargin()\sroutine\sof\sR-Tree\sslightly\ssmaller\sand\sfaster\swhile\salso\nfixing\sa\sharmless\scompiler\swarning. +D 2017-02-07T12:58:38.284 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 76311736628e87fd780dfbbbf5813bf1f9bf2329 +F ext/rtree/rtree.c 773fbf1217a12607a5898eeb5193974b6d2bdad4 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1554,8 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f77ee9e941f22b0b2e88871df4466fdde9cde131 -Q -ec8ff892ac9c6a8f81bcf69f1933f4bb69faa743 -R ebea043e9962974403d836188c02b08f +P 0611770d6a2fcb9fa0bedee19df9916820ae8f3b +R b3ac1d676d5ca604a8be68bb866fbceb U drh -Z a87f51011f00704857e3612bb8043da6 +Z 3ad4f65e4d593a16597f73c400a250b4 diff --git a/manifest.uuid b/manifest.uuid index 1b41d3e917..865247349a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0611770d6a2fcb9fa0bedee19df9916820ae8f3b \ No newline at end of file +07fe6228208684d579c4f6c334c90eb6262a9233 \ No newline at end of file From 364ca6a90e59f6453ff4b136fbce617f8743649a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 13:51:48 +0000 Subject: [PATCH 126/292] Fix SQLITE_BYTEORDER #defines in R-Tree when compiled separately from the amalgamation. FossilOrigin-Name: a136609c98ed3cc673c5a3c2578d49db3f2518d1 --- ext/rtree/rtree.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index e898e34b79..a288272517 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -408,13 +408,13 @@ struct RtreeMatchArg { defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) # define SQLITE_BYTEORDER 1234 -#endif -#if (defined(sparc) || defined(__ppc__)) \ +#elif (defined(sparc) || defined(__ppc__)) \ && !defined(SQLITE_RUNTIME_BYTEORDER) # define SQLITE_BYTEORDER 4321 -#endif +#else # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ #endif +#endif /* What version of MSVC is being used. 0 means MSVC is not being used */ diff --git a/manifest b/manifest index 255e0df908..1e7c95b554 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\scellMargin()\sroutine\sof\sR-Tree\sslightly\ssmaller\sand\sfaster\swhile\salso\nfixing\sa\sharmless\scompiler\swarning. -D 2017-02-07T12:58:38.284 +C Fix\sSQLITE_BYTEORDER\s#defines\sin\sR-Tree\swhen\scompiled\sseparately\sfrom\sthe\namalgamation. +D 2017-02-07T13:51:48.651 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 773fbf1217a12607a5898eeb5193974b6d2bdad4 +F ext/rtree/rtree.c 97b6560be68c20d27169ec14b805609f3ca4d90f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0611770d6a2fcb9fa0bedee19df9916820ae8f3b -R b3ac1d676d5ca604a8be68bb866fbceb +P 07fe6228208684d579c4f6c334c90eb6262a9233 +R fb7b4c31979a0a5dd9190e2f71115220 U drh -Z 3ad4f65e4d593a16597f73c400a250b4 +Z e9cc05bda60d84b045dd690e40ba6bd5 diff --git a/manifest.uuid b/manifest.uuid index 865247349a..cbfdaf9434 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07fe6228208684d579c4f6c334c90eb6262a9233 \ No newline at end of file +a136609c98ed3cc673c5a3c2578d49db3f2518d1 \ No newline at end of file From 821fad51b4b93980dca0b886b760645d097b41ab Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Feb 2017 14:22:39 +0000 Subject: [PATCH 127/292] Fix a test case to account for the fact that sqlite3_blob_reopen() now returns SQLITE_IOERR_NOMEM instead of SQLITE_NOMEM if an OOM occurs in the VFS layer. FossilOrigin-Name: e11cc52389f61f6421179281877b119c02286121 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/incrblobfault.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1e7c95b554..eb4d617f5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sSQLITE_BYTEORDER\s#defines\sin\sR-Tree\swhen\scompiled\sseparately\sfrom\sthe\namalgamation. -D 2017-02-07T13:51:48.651 +C Fix\sa\stest\scase\sto\saccount\sfor\sthe\sfact\sthat\ssqlite3_blob_reopen()\snow\sreturns\nSQLITE_IOERR_NOMEM\sinstead\sof\sSQLITE_NOMEM\sif\san\sOOM\soccurs\sin\sthe\sVFS\slayer. +D 2017-02-07T14:22:39.073 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -851,7 +851,7 @@ F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 F test/incrblob4.test 21a52a6843a56cdcce968c6a86b72a7066d0e6ba F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 -F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e +F test/incrblobfault.test 74dd8ac108304cea0b4a0df6df63a1567e558758 F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07fe6228208684d579c4f6c334c90eb6262a9233 -R fb7b4c31979a0a5dd9190e2f71115220 -U drh -Z e9cc05bda60d84b045dd690e40ba6bd5 +P a136609c98ed3cc673c5a3c2578d49db3f2518d1 +R b322167caccfbef7ed8b051ff68195b2 +U dan +Z ce20f1c1fe7e20be76a40b05210db04c diff --git a/manifest.uuid b/manifest.uuid index cbfdaf9434..dd14d6d04e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a136609c98ed3cc673c5a3c2578d49db3f2518d1 \ No newline at end of file +e11cc52389f61f6421179281877b119c02286121 \ No newline at end of file diff --git a/test/incrblobfault.test b/test/incrblobfault.test index 0c1d93d460..10c2c8ecb4 100644 --- a/test/incrblobfault.test +++ b/test/incrblobfault.test @@ -52,7 +52,7 @@ do_faultsim_test 2 -prep { error [sqlite3_errmsg db] } } -test { - faultsim_test_result {1 {no such rowid: -1}} + faultsim_test_result {1 {no such rowid: -1}} {1 {disk I/O error}} close $::blob } From 9fcb6ddc1e4eba176fdd32cb7b8968ba81e9c629 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Feb 2017 14:45:18 +0000 Subject: [PATCH 128/292] Fix a build problem affecting non-amalgamation rtree builds. FossilOrigin-Name: bb7f445ba1df53cd4a169612b18fc533016102b7 --- ext/rtree/rtree.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index a288272517..1e3e65ecc9 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -68,6 +68,7 @@ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; diff --git a/manifest b/manifest index eb4d617f5c..4651930793 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sto\saccount\sfor\sthe\sfact\sthat\ssqlite3_blob_reopen()\snow\sreturns\nSQLITE_IOERR_NOMEM\sinstead\sof\sSQLITE_NOMEM\sif\san\sOOM\soccurs\sin\sthe\sVFS\slayer. -D 2017-02-07T14:22:39.073 +C Fix\sa\sbuild\sproblem\saffecting\snon-amalgamation\srtree\sbuilds. +D 2017-02-07T14:45:18.924 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 97b6560be68c20d27169ec14b805609f3ca4d90f +F ext/rtree/rtree.c d92be99e3a662e1934658201f0fc7ad51b44325a F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a136609c98ed3cc673c5a3c2578d49db3f2518d1 -R b322167caccfbef7ed8b051ff68195b2 +P e11cc52389f61f6421179281877b119c02286121 +R ba6eaa451f165f44792dd902ec15651c U dan -Z ce20f1c1fe7e20be76a40b05210db04c +Z 82882b9c6c63f305c028e634c6349ae2 diff --git a/manifest.uuid b/manifest.uuid index dd14d6d04e..f33ceba2a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e11cc52389f61f6421179281877b119c02286121 \ No newline at end of file +bb7f445ba1df53cd4a169612b18fc533016102b7 \ No newline at end of file From 2e9dceb432cca5f58f4c4b474c9348038492e56f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Feb 2017 16:15:48 +0000 Subject: [PATCH 129/292] Update an error message in corruptC.test. FossilOrigin-Name: 722e57fe61162a82b8001c7b1d3e06174caa90b9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/corruptC.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4651930793..0378ad6c62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbuild\sproblem\saffecting\snon-amalgamation\srtree\sbuilds. -D 2017-02-07T14:45:18.924 +C Update\san\serror\smessage\sin\scorruptC.test. +D 2017-02-07T16:15:48.783 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -607,7 +607,7 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 53e56dafd180addcdadb402244b8cb9771d2ba26 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 4ef10844eba5213bd262f4d96784d7fcda114afe +F test/corruptC.test 46ec43bd90a02fd7b37ad8a7a949c55aa5717f89 F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 82ccf4f8f543fdbedd4aa42c709cb077f7374c62 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 @@ -1554,7 +1554,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e11cc52389f61f6421179281877b119c02286121 -R ba6eaa451f165f44792dd902ec15651c +P bb7f445ba1df53cd4a169612b18fc533016102b7 +R 1228beaef5172fbebdd4d0250089ef1e U dan -Z 82882b9c6c63f305c028e634c6349ae2 +Z cdab5e1aea1876be0fe4dd420440969d diff --git a/manifest.uuid b/manifest.uuid index f33ceba2a3..9aa189d3e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb7f445ba1df53cd4a169612b18fc533016102b7 \ No newline at end of file +722e57fe61162a82b8001c7b1d3e06174caa90b9 \ No newline at end of file diff --git a/test/corruptC.test b/test/corruptC.test index e2fb1f33c4..f404e4fb5b 100644 --- a/test/corruptC.test +++ b/test/corruptC.test @@ -164,7 +164,7 @@ do_test corruptC-2.5 { catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;} catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -Page 4: btreeInitPage() returns error code 11}}} +On tree page 4 cell 19: Extends off end of page}}} # {0 {{*** in database main *** # Corruption detected in cell 710 on page 4 From b9e9bc1de62a877a3c18baede844dd07d7ed7775 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 17:36:12 +0000 Subject: [PATCH 130/292] Add the initial version of the "dbselftest" utility program. FossilOrigin-Name: 1fcac8365e0f7bcfd55442d718da6626c864d45a --- Makefile.in | 5 + main.mk | 7 +- manifest | 17 +- manifest.uuid | 2 +- test/dbselftest.c | 605 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 626 insertions(+), 10 deletions(-) create mode 100644 test/dbselftest.c diff --git a/Makefile.in b/Makefile.in index d5fa831e6b..740b49c08f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1185,6 +1185,11 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) +DBSELFTEST_OPT += -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 + +dbselftest$(TEXE): $(TOP)/test/dbselftest.c sqlite3.c + $(LTLINK) $(DBSELFTEST_OPT) -o $@ $(TOP)/test/dbselftest.c sqlite3.c $(TLIBS) + rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) diff --git a/main.mk b/main.mk index 17e6468e10..85ed76a143 100644 --- a/main.mk +++ b/main.mk @@ -479,6 +479,8 @@ FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +DBSELFTEST_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION +DBSELFTEST_OPT += -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here @@ -898,7 +900,10 @@ speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c $(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c - $(TCCX) -I. $(KV+OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) + $(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) + +dbselftest$(EXE): $(TOP)/test/dbselftest.c sqlite3.c + $(TCCX) -I. $(DBSELFTEST_OPT) -o dbselftest$(EXE) $(TOP)/test/dbselftest.c sqlite3.c $(THREADLIB) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ diff --git a/manifest b/manifest index 0378ad6c62..2432d3968f 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Update\san\serror\smessage\sin\scorruptC.test. -D 2017-02-07T16:15:48.783 -F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc +C Add\sthe\sinitial\sversion\sof\sthe\s"dbselftest"\sutility\sprogram. +D 2017-02-07T17:36:12.948 +F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -318,7 +318,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk afc52937b4e5fe08678e8d5a4fe4487d44e3bc61 +F main.mk 08e0b151abb46353c677ac4974b50c6077f85eee F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -636,6 +636,7 @@ F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc F test/dbfuzz.c 8cc2bdb818b4483a052f9f80f96be74cbd9a6e1d +F test/dbselftest.c eeb95d09932b4dc79e5886bd8cf5d812e7d28d94 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1554,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb7f445ba1df53cd4a169612b18fc533016102b7 -R 1228beaef5172fbebdd4d0250089ef1e -U dan -Z cdab5e1aea1876be0fe4dd420440969d +P 722e57fe61162a82b8001c7b1d3e06174caa90b9 +R 9f073837e9cdb1ff055e4a2a2994d017 +U drh +Z f48b3d2fea77878a00660f4925462068 diff --git a/manifest.uuid b/manifest.uuid index 9aa189d3e7..e2570868a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -722e57fe61162a82b8001c7b1d3e06174caa90b9 \ No newline at end of file +1fcac8365e0f7bcfd55442d718da6626c864d45a \ No newline at end of file diff --git a/test/dbselftest.c b/test/dbselftest.c new file mode 100644 index 0000000000..1c6f8d8b9f --- /dev/null +++ b/test/dbselftest.c @@ -0,0 +1,605 @@ +/* +** 2017-02-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This program implements an SQLite database self-verification utility. +** Usage: +** +** dbselftest DATABASE +** +** This program reads the "selftest" table in DATABASE, in rowid order, +** and runs each of the tests described there, reporting results at the +** end. +** +** The intent of this program is to have a set of test database files that +** can be run using future versions of SQLite in order to verify that +** legacy database files continue to be readable. In other words, the +** intent is to confirm that there have been no breaking changes in the +** file format. The program can also be used to verify that database files +** are fully compatible between different architectures. +** +** The selftest table looks like this: +** +** CREATE TABLE selftest ( +** id INTEGER PRIMARY KEY, -- Run tests in ascending order +** op TEXT, -- "test", "regexp", "print", etc. +** cmdtxt TEXT, -- Usually the SQL to be run +** expected TEXT -- Expected results +** ); +** +*/ +#include +#include +#include +#include +#include +#include "sqlite3.h" + +static const char zHelp[] = + "Usage: dbselftest [OPTIONS] DBFILE\n" + "\n" + " --init Create the selftest table\n" + " -q Suppress most output. Errors only\n" + " -v Show extra output\n" +; + + +/****************************************************************************** +** The following code from ext/misc/sha1.c +** +** Context for the SHA1 hash +*/ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + + +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut /* Store hexadecimal hash here */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; +} + +/* +** Implementation of the sha1(X) function. +** +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hash without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + char zOut[44]; + + assert( argc==1 ); + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + assert( argc==1 ); + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} +/* End of ext/misc/sha1.c +******************************************************************************/ + +/* How much output to display */ +#define VOLUME_MIN 0 +#define VOLUME_OFF 0 +#define VOLUME_ERROR_ONLY 1 +#define VOLUME_LOW 2 +#define VOLUME_ECHO 3 +#define VOLUME_VERBOSE 4 +#define VOLUME_MAX 4 + +/* A string accumulator +*/ +typedef struct Str { + char *z; /* Accumulated text */ + int n; /* Bytes of z[] used so far */ + int nAlloc; /* Bytes allocated for z[] */ +} Str; + +/* Append text to the Str object +*/ +static void strAppend(Str *p, const char *z){ + int n = (int)strlen(z); + if( p->n+n >= p->nAlloc ){ + p->nAlloc += p->n+n + 100; + p->z = sqlite3_realloc(p->z, p->nAlloc); + if( z==0 ){ + printf("Could not allocate %d bytes\n", p->nAlloc); + exit(1); + } + } + memcpy(p->z+n, z, n+1); + p->n += n; +} + +/* This is an sqlite3_exec() callback that will capture all +** output in a Str. +** +** Columns are separated by ",". Rows are separated by "|". +*/ +static int execCallback(void *pStr, int argc, char **argv, char **colv){ + int i; + Str *p = (Str*)pStr; + if( p->n ) strAppend(p, "|"); + for(i=0; i0 ) strAppend(p, ","); + strAppend(p, z); + } + return 0; +} + +int main(int argc, char **argv){ + int eVolume = VOLUME_LOW; /* How much output to display */ + const char *zDb = 0; /* Name of the database file */ + int doInit = 0; /* True if --init is present */ + sqlite3 *db = 0; /* Open database connection */ + int rc; /* Return code from API calls */ + char *zErrMsg = 0; /* An error message return */ + char **azTest; /* Content of the selftest table */ + int nRow = 0, nCol = 0; /* Rows and columns in azTest[] */ + int i; /* Loop counter */ + int nErr = 0; /* Number of errors */ + Str str; /* Result accumulator */ + + for(i=1; iVOLUME_MIN) eVolume--; + }else + if( strcmp(z, "-v")==0 ){ + if( eVolume=VOLUME_LOW ){ + printf("SQLite %s\n", sqlite3_sourceid()); + } + memset(&str, 0, sizeof(str)); + strAppend(&str, "\n"); + for(i=1; i<=nRow; i++){ + int tno = atoi(azTest[i*nCol]); + const char *zOp = azTest[i*nCol+1]; + const char *zSql = azTest[i*nCol+2]; + const char *zAns = azTest[i*nCol+3]; + + if( eVolume>=VOLUME_ECHO ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + if( eVolume>=VOLUME_LOW ){ + printf("%s\n", zSql); + } + }else + if( strcmp(zOp,"run")==0 ){ + str.n = 0; + str.z[0] = 0; + zErrMsg = 0; + rc = sqlite3_exec(db, zSql, execCallback, &str, &zErrMsg); + if( eVolume>=VOLUME_VERBOSE ){ + printf("Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); + } + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: Expected: [%s]\n", tno, zAns); + printf("%d: Got: [%s]\n", tno, str.z); + } + } + }else + { + printf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + return 1; + } + } + sqlite3_free_table(azTest); + sqlite3_close(db); + if( eVolume>=VOLUME_LOW || (nErr>0 && eVolume>=VOLUME_ERROR_ONLY) ){ + printf("%d errors out of %d tests\n", nErr, nRow); + } + return nErr; +} From a7fc253d42ba787c60c33dc6e760fad5796ba817 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Feb 2017 19:23:51 +0000 Subject: [PATCH 131/292] Update a test parameter in malloc5.test to account for the increase in default lookaside buffer size. FossilOrigin-Name: be82d5ae20ba62a165bdc28766a8dc8049abcac6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/malloc5.test | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2432d3968f..36862460f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sinitial\sversion\sof\sthe\s"dbselftest"\sutility\sprogram. -D 2017-02-07T17:36:12.948 +C Update\sa\stest\sparameter\sin\smalloc5.test\sto\saccount\sfor\sthe\sincrease\sin\sdefault\nlookaside\sbuffer\ssize. +D 2017-02-07T19:23:51.831 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -928,7 +928,7 @@ F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 -F test/malloc5.test 02ed7c5313f0a68d95f2dfca8c8962132bd1f04b +F test/malloc5.test 192d0b5ee1d023a07d5164c34cf47d010da73da1 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 722e57fe61162a82b8001c7b1d3e06174caa90b9 -R 9f073837e9cdb1ff055e4a2a2994d017 -U drh -Z f48b3d2fea77878a00660f4925462068 +P 1fcac8365e0f7bcfd55442d718da6626c864d45a +R d6f712b168d87479bbb4e9d5c8c40830 +U dan +Z 0f2ec469c3345d822f702c8d7a665aee diff --git a/manifest.uuid b/manifest.uuid index e2570868a3..7b0c1f1a48 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fcac8365e0f7bcfd55442d718da6626c864d45a \ No newline at end of file +be82d5ae20ba62a165bdc28766a8dc8049abcac6 \ No newline at end of file diff --git a/test/malloc5.test b/test/malloc5.test index 8f0db04c2a..8114005e8b 100644 --- a/test/malloc5.test +++ b/test/malloc5.test @@ -236,12 +236,12 @@ do_test malloc5-4.2 { db eval {PRAGMA cache_size=1} db cache flush sqlite3_release_memory - sqlite3_soft_heap_limit 100000 + sqlite3_soft_heap_limit 200000 sqlite3_memory_highwater 1 execsql {SELECT * FROM abc} set nMaxBytes [sqlite3_memory_highwater 1] puts -nonewline " (Highwater mark: $nMaxBytes) " - expr $nMaxBytes <= 110000 + expr $nMaxBytes <= 210000 } {1} do_test malloc5-4.3 { # Check that the content of table abc is at least roughly as expected. From 18b20c981d47127ffeb8b38e6a654caea650f25d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Feb 2017 19:36:14 +0000 Subject: [PATCH 132/292] Omit fts5fault1.test from the inmemory_journal permutation. FossilOrigin-Name: cb1e83f9583bf93ce7583d9f5e97272e2d43cfb8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 36862460f4..854ed84459 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\stest\sparameter\sin\smalloc5.test\sto\saccount\sfor\sthe\sincrease\sin\sdefault\nlookaside\sbuffer\ssize. -D 2017-02-07T19:23:51.831 +C Omit\sfts5fault1.test\sfrom\sthe\sinmemory_journal\spermutation. +D 2017-02-07T19:36:14.794 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -1015,7 +1015,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cd0b7bc04bf5e50d3a993d24c834d591f7d4f5fe +F test/permutations.test 46ad98770d3c04ce2418cab2cc527647bfe961a7 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1fcac8365e0f7bcfd55442d718da6626c864d45a -R d6f712b168d87479bbb4e9d5c8c40830 +P be82d5ae20ba62a165bdc28766a8dc8049abcac6 +R 4c6aa653d353c19152ed7b2aaba4bc83 U dan -Z 0f2ec469c3345d822f702c8d7a665aee +Z 450aa5587fc5eb1e148efd8100bf109e diff --git a/manifest.uuid b/manifest.uuid index 7b0c1f1a48..026b45bd03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be82d5ae20ba62a165bdc28766a8dc8049abcac6 \ No newline at end of file +cb1e83f9583bf93ce7583d9f5e97272e2d43cfb8 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index f8880da225..7f81dcbb27 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -754,6 +754,10 @@ test_suite "inmemory_journal" -description { # This test assumes a journal file is created on disk. delete_db.test + + # This test depends on a successful recovery from the pager error + # state. Which is not possible with an in-memory journal + fts5fault1.test }] ifcapable mem3 { From cbc65e5f4f3fc24a0cfe17eb723f4439cb7bc01e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 20:51:38 +0000 Subject: [PATCH 133/292] The dbselftest utility now generates hashes in the selftest table with --init. It also accepts multiple database files on the command-line. FossilOrigin-Name: e68829c9bbc69bf4a0dc057e0a6e977f2fac79be --- manifest | 14 +- manifest.uuid | 2 +- test/dbselftest.c | 497 +++++++++++++++++++++++++++++++--------------- 3 files changed, 347 insertions(+), 166 deletions(-) diff --git a/manifest b/manifest index 854ed84459..a37a9e38bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sfts5fault1.test\sfrom\sthe\sinmemory_journal\spermutation. -D 2017-02-07T19:36:14.794 +C The\sdbselftest\sutility\snow\sgenerates\shashes\sin\sthe\sselftest\stable\swith\s--init.\nIt\salso\saccepts\smultiple\sdatabase\sfiles\son\sthe\scommand-line. +D 2017-02-07T20:51:38.461 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -636,7 +636,7 @@ F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc F test/dbfuzz.c 8cc2bdb818b4483a052f9f80f96be74cbd9a6e1d -F test/dbselftest.c eeb95d09932b4dc79e5886bd8cf5d812e7d28d94 +F test/dbselftest.c 4bf86fe04dc2d0d58dabc5e6b7a06fa4ef7827ea F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be82d5ae20ba62a165bdc28766a8dc8049abcac6 -R 4c6aa653d353c19152ed7b2aaba4bc83 -U dan -Z 450aa5587fc5eb1e148efd8100bf109e +P cb1e83f9583bf93ce7583d9f5e97272e2d43cfb8 +R 64e5750ab44994d5b187d93e5c4a1318 +U drh +Z f3a9d081d357071c2c4fc7a23224549e diff --git a/manifest.uuid b/manifest.uuid index 026b45bd03..6596630605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb1e83f9583bf93ce7583d9f5e97272e2d43cfb8 \ No newline at end of file +e68829c9bbc69bf4a0dc057e0a6e977f2fac79be \ No newline at end of file diff --git a/test/dbselftest.c b/test/dbselftest.c index 1c6f8d8b9f..d8a9c331de 100644 --- a/test/dbselftest.c +++ b/test/dbselftest.c @@ -13,7 +13,7 @@ ** This program implements an SQLite database self-verification utility. ** Usage: ** -** dbselftest DATABASE +** dbselftest DATABASE ... ** ** This program reads the "selftest" table in DATABASE, in rowid order, ** and runs each of the tests described there, reporting results at the @@ -44,7 +44,7 @@ #include "sqlite3.h" static const char zHelp[] = - "Usage: dbselftest [OPTIONS] DBFILE\n" + "Usage: dbselftest [OPTIONS] DBFILE ...\n" "\n" " --init Create the selftest table\n" " -q Suppress most output. Errors only\n" @@ -294,6 +294,101 @@ static void sha1Func( sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } +/* +** Run a prepared statement and compute the SHA1 hash on the +** result rows. +*/ +static void sha1RunStatement(SHA1Context *pCtx, sqlite3_stmt *pStmt){ + int nCol = sqlite3_column_count(pStmt); + const char *z = sqlite3_sql(pStmt); + int n = (int)strlen(z); + + hash_step_vformat(pCtx,"S%d:",n); + hash_step(pCtx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int i; + hash_step(pCtx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(pCtx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(pCtx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(pCtx,"T%d:",n2); + hash_step(pCtx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(pCtx,"B%d:",n2); + hash_step(pCtx, z2, n2); + break; + } + } + } + } +} + +/* +** Run one or more statements of SQL. Compute a SHA1 hash of the output. +*/ +static int sha1Exec( + sqlite3 *db, /* Run against this database connection */ + const char *zSql, /* The SQL to be run */ + char *zOut /* Store the SHA1 hash as hexadecimal in this buffer */ +){ + sqlite3_stmt *pStmt = 0; /* A prepared statement */ + int rc; /* Result of an API call */ + SHA1Context cx; /* The SHA1 hash context */ + + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + sqlite3_finalize(pStmt); + return rc; + } + sha1RunStatement(&cx, pStmt); + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut); + return SQLITE_OK; +} + /* ** Implementation of the sha1_query(SQL) function. ** @@ -314,11 +409,7 @@ static void sha1QueryFunc( sqlite3 *db = sqlite3_context_db_handle(context); const char *zSql = (const char*)sqlite3_value_text(argv[0]); sqlite3_stmt *pStmt = 0; - int nCol; /* Number of columns in the result set */ - int i; /* Loop counter */ int rc; - int n; - const char *z; SHA1Context cx; char zOut[44]; @@ -342,66 +433,7 @@ static void sha1QueryFunc( sqlite3_free(zMsg); return; } - nCol = sqlite3_column_count(pStmt); - z = sqlite3_sql(pStmt); - n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); - hash_step(&cx,(unsigned char*)z,n); - - /* Compute a hash over the result of the query */ - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - hash_step(&cx,(const unsigned char*)"R",1); - for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - hash_step(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - hash_step(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); - hash_step(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); - hash_step(&cx, z2, n2); - break; - } - } - } - } + sha1RunStatement(&cx, pStmt); sqlite3_finalize(pStmt); } hash_finish(&cx, zOut); @@ -439,7 +471,7 @@ static void strAppend(Str *p, const char *z){ exit(1); } } - memcpy(p->z+n, z, n+1); + memcpy(p->z+p->n, z, n+1); p->n += n; } @@ -461,9 +493,138 @@ static int execCallback(void *pStr, int argc, char **argv, char **colv){ return 0; } +/* +** Run an SQL statement constructing using sqlite3_vmprintf(). +** Return the number of errors. +*/ +static int runSql(sqlite3 *db, const char *zFormat, ...){ + char *zSql; + char *zErr = 0; + int rc; + int nErr = 0; + va_list ap; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + printf("Out of memory\n"); + exit(1); + } + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc || zErr ){ + printf("SQL error in [%s]: code=%d: %s\n", zSql, rc, zErr); + nErr++; + } + sqlite3_free(zSql); + return nErr; +} + +/* +** Generate a prepared statement using a formatted string. +*/ +static sqlite3_stmt *prepareSql(sqlite3 *db, const char *zFormat, ...){ + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + va_list ap; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + printf("Out of memory\n"); + exit(1); + } + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ){ + printf("SQL error in [%s]: code=%d: %s\n", zSql, rc, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + pStmt = 0; + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Construct the standard selftest configuration for the database. +*/ +static int buildSelftestTable(sqlite3 *db){ + int rc; + sqlite3_stmt *pStmt; + int tno = 110; + char *zSql; + char zHash[50]; + + rc = runSql(db, + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY, -- test number\n" + " op TEXT, -- what kind of test\n" + " sql TEXT, -- SQL text for the test\n" + " ans TEXT -- expected answer\n" + ");" + "INSERT INTO selftest" + " VALUES(100,'memo','Hashes generated using --init',NULL);" + ); + if( rc ) return 1; + tno = 110; + zSql = "SELECT type,name,tbl_name,sql FROM sqlite_master"; + sha1Exec(db, zSql, zHash); + rc = runSql(db, + "INSERT INTO selftest(tno,op,sql,ans)" + " VALUES(%d,'sha1',%Q,%Q)", tno, zSql, zHash); + tno += 10; + pStmt = prepareSql(db, + "SELECT lower(name) FROM sqlite_master" + " WHERE type='table' AND sql NOT GLOB 'CREATE VIRTUAL*'" + " AND name<>'selftest'" + " ORDER BY 1"); + if( pStmt==0 ) return 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + zSql = sqlite3_mprintf("SELECT * FROM \"%w\" NOT INDEXED", + sqlite3_column_text(pStmt, 0)); + if( zSql==0 ){ + printf("Of of memory\n"); + exit(1); + } + sha1Exec(db, zSql, zHash); + rc = runSql(db, + "INSERT INTO selftest(tno,op,sql,ans)" + " VALUES(%d,'sha1',%Q,%Q)", tno, zSql, zHash); + tno += 10; + sqlite3_free(zSql); + if( rc ) break; + } + sqlite3_finalize(pStmt); + if( rc ) return 1; + rc = runSql(db, + "INSERT INTO selftest(tno,op,sql,ans)" + " VALUES(%d,'run','PRAGMA integrity_check','ok');", tno); + if( rc ) return 1; + return rc; +} + +/* +** Return true if the named table exists +*/ +static int tableExists(sqlite3 *db, const char *zTab){ + return sqlite3_table_column_metadata(db, "main", zTab, 0, 0, 0, 0, 0, 0) + == SQLITE_OK; +} + +/* +** Default selftest table content, for use when there is no selftest table +*/ +static char *azDefaultTest[] = { + 0, 0, 0, 0, + "0", "memo", "Missing SELFTEST table - default checks only", "", + "1", "run", "PRAGMA integrity_check", "ok" +}; + int main(int argc, char **argv){ int eVolume = VOLUME_LOW; /* How much output to display */ - const char *zDb = 0; /* Name of the database file */ + const char **azDb = 0; /* Name of the database file */ + int nDb = 0; /* Number of database files to check */ int doInit = 0; /* True if --init is present */ sqlite3 *db = 0; /* Open database connection */ int rc; /* Return code from API calls */ @@ -472,7 +633,9 @@ int main(int argc, char **argv){ int nRow = 0, nCol = 0; /* Rows and columns in azTest[] */ int i; /* Loop counter */ int nErr = 0; /* Number of errors */ + int iDb; /* Loop counter for databases */ Str str; /* Result accumulator */ + int nTest = 0; /* Number of tests run */ for(i=1; i=VOLUME_LOW ){ @@ -553,53 +678,109 @@ int main(int argc, char **argv){ } memset(&str, 0, sizeof(str)); strAppend(&str, "\n"); - for(i=1; i<=nRow; i++){ - int tno = atoi(azTest[i*nCol]); - const char *zOp = azTest[i*nCol+1]; - const char *zSql = azTest[i*nCol+2]; - const char *zAns = azTest[i*nCol+3]; - - if( eVolume>=VOLUME_ECHO ){ - char *zQuote = sqlite3_mprintf("%q", zSql); - printf("%d: %s %s\n", tno, zOp, zSql); - sqlite3_free(zQuote); - } - if( strcmp(zOp,"memo")==0 ){ - if( eVolume>=VOLUME_LOW ){ - printf("%s\n", zSql); - } - }else - if( strcmp(zOp,"run")==0 ){ - str.n = 0; - str.z[0] = 0; - zErrMsg = 0; - rc = sqlite3_exec(db, zSql, execCallback, &str, &zErrMsg); - if( eVolume>=VOLUME_VERBOSE ){ - printf("Result: %s\n", str.z); - } - if( rc || zErrMsg ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); - } - sqlite3_free(zErrMsg); - }else if( strcmp(zAns,str.z)!=0 ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: Expected: [%s]\n", tno, zAns); - printf("%d: Got: [%s]\n", tno, str.z); - } - } - }else - { - printf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + for(iDb=0; iDb=VOLUME_ECHO ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + if( eVolume>=VOLUME_LOW ){ + printf("%s: %s\n", azDb[iDb], zSql); + } + }else + if( strcmp(zOp,"sha1")==0 ){ + char zOut[44]; + rc = sha1Exec(db, zSql, zOut); + nTest++; + if( eVolume>=VOLUME_VERBOSE ){ + printf("Result: %s\n", zOut); + } + if( rc ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: error-code-%d: %s\n", tno, rc, sqlite3_errmsg(db)); + } + }else if( strcmp(zAns,zOut)!=0 ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: Expected: [%s]\n", tno, zAns); + printf("%d: Got: [%s]\n", tno, zOut); + } + } + }else + if( strcmp(zOp,"run")==0 ){ + str.n = 0; + str.z[0] = 0; + zErrMsg = 0; + rc = sqlite3_exec(db, zSql, execCallback, &str, &zErrMsg); + nTest++; + if( eVolume>=VOLUME_VERBOSE ){ + printf("Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); + } + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + if( eVolume>=VOLUME_ERROR_ONLY ){ + printf("%d: Expected: [%s]\n", tno, zAns); + printf("%d: Got: [%s]\n", tno, str.z); + } + } + }else + { + printf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + return 1; + } + } + if( azTest!=azDefaultTest ) sqlite3_free_table(azTest); } - sqlite3_free_table(azTest); - sqlite3_close(db); if( eVolume>=VOLUME_LOW || (nErr>0 && eVolume>=VOLUME_ERROR_ONLY) ){ - printf("%d errors out of %d tests\n", nErr, nRow); + printf("%d errors out of %d tests on %d databases\n", nErr, nTest, nDb); } return nErr; } From b67e1754489a6701271868564ed4ba7e05f5ce5e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 20:57:00 +0000 Subject: [PATCH 134/292] Fix harmless compiler warnings in kvtest.c FossilOrigin-Name: db6b39937dd9d6fcd5fcc582c8dd529caad755da --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/kvtest.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a37a9e38bf..2284814a6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sdbselftest\sutility\snow\sgenerates\shashes\sin\sthe\sselftest\stable\swith\s--init.\nIt\salso\saccepts\smultiple\sdatabase\sfiles\son\sthe\scommand-line. -D 2017-02-07T20:51:38.461 +C Fix\sharmless\scompiler\swarnings\sin\skvtest.c +D 2017-02-07T20:57:00.794 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -904,7 +904,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c 156281b4ed688e9aabf1fbcdc5dec0487b267026 +F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb1e83f9583bf93ce7583d9f5e97272e2d43cfb8 -R 64e5750ab44994d5b187d93e5c4a1318 +P e68829c9bbc69bf4a0dc057e0a6e977f2fac79be +R e8e95a0beaf31ff9600f2388a71f8dcd U drh -Z f3a9d081d357071c2c4fc7a23224549e +Z 62b1a410058a970df83f462c97cdf484 diff --git a/manifest.uuid b/manifest.uuid index 6596630605..473f751fff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e68829c9bbc69bf4a0dc057e0a6e977f2fac79be \ No newline at end of file +db6b39937dd9d6fcd5fcc582c8dd529caad755da \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 77ee02a4ff..4fa733f1a7 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -473,13 +473,13 @@ static unsigned char *readFile(const char *zName, int *pnByte){ if( in==0 ) return 0; pBuf = sqlite3_malloc64( nIn ); if( pBuf==0 ) return 0; - nRead = fread(pBuf, nIn, 1, in); + nRead = fread(pBuf, (size_t)nIn, 1, in); fclose(in); if( nRead!=1 ){ sqlite3_free(pBuf); return 0; } - if( pnByte ) *pnByte = nIn; + if( pnByte ) *pnByte = (int)nIn; return pBuf; } From 134e527091d86f164b42a47b3777ac71a335ff18 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 21:00:44 +0000 Subject: [PATCH 135/292] Fix harmless compiler warnings in dbfuzz. FossilOrigin-Name: 61242267824135a9d5438ec15e3352a2f21dc2fc --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/dbfuzz.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2284814a6d..cf619e924e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\skvtest.c -D 2017-02-07T20:57:00.794 +C Fix\sharmless\scompiler\swarnings\sin\sdbfuzz. +D 2017-02-07T21:00:44.802 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 @@ -635,7 +635,7 @@ F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc -F test/dbfuzz.c 8cc2bdb818b4483a052f9f80f96be74cbd9a6e1d +F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbselftest.c 4bf86fe04dc2d0d58dabc5e6b7a06fa4ef7827ea F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e68829c9bbc69bf4a0dc057e0a6e977f2fac79be -R e8e95a0beaf31ff9600f2388a71f8dcd +P db6b39937dd9d6fcd5fcc582c8dd529caad755da +R f5b20543ad5a9de57f328e5cb19dc17f U drh -Z 62b1a410058a970df83f462c97cdf484 +Z 556fccfd2bf9ea9a682f73b7b4dda1ae diff --git a/manifest.uuid b/manifest.uuid index 473f751fff..db9339c4af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db6b39937dd9d6fcd5fcc582c8dd529caad755da \ No newline at end of file +61242267824135a9d5438ec15e3352a2f21dc2fc \ No newline at end of file diff --git a/test/dbfuzz.c b/test/dbfuzz.c index 2de97c35cb..ca1c6ea217 100644 --- a/test/dbfuzz.c +++ b/test/dbfuzz.c @@ -458,7 +458,7 @@ static void StrAppend(Str *p, const char *z){ sqlite3_uint64 nNew; if( p->oomErr ) return; nNew = p->nAlloc*2 + 100 + n; - zNew = sqlite3_realloc(p->z, nNew); + zNew = sqlite3_realloc64(p->z, nNew); if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); @@ -468,7 +468,7 @@ static void StrAppend(Str *p, const char *z){ p->z = zNew; p->nAlloc = nNew; } - memcpy(p->z + p->n, z, n); + memcpy(p->z + p->n, z, (int)n); p->n += n; p->z[p->n] = 0; } @@ -647,7 +647,7 @@ static void runSql(sqlite3 *db, const char *zSql, unsigned runFlags){ int main(int argc, char **argv){ int i; /* Loop counter */ int nDb = 0; /* Number of databases to fuzz */ - const char **azDb = 0; /* Names of the databases (limit: 20) */ + char **azDb = 0; /* Names of the databases (limit: 20) */ int verboseFlag = 0; /* True for extra output */ int noLookaside = 0; /* Disable lookaside if true */ int vdbeLimitFlag = 0; /* Stop after 100,000 VDBE ops */ @@ -660,7 +660,7 @@ int main(int argc, char **argv){ unsigned runFlags = 0; /* Flags passed to runSql */ for(i=1; i Date: Tue, 7 Feb 2017 21:09:34 +0000 Subject: [PATCH 136/292] MSVC makefile enhancments for several command line tools and the session extensions. FossilOrigin-Name: fc1dfe870279531d77ffb18a9ca4b4dcbc7aab8d --- Makefile.msc | 41 ++++++++++++++++++++++++++++------------- autoconf/Makefile.msc | 2 +- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 2 +- tool/mksqlite3h.tcl | 12 ++++++++++-- 6 files changed, 51 insertions(+), 28 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 4a29f913c8..8b98fe4cac 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1512,6 +1512,10 @@ FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 # Standard options to testfixture. # @@ -1554,7 +1558,7 @@ $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def # <> @@ -1581,6 +1585,9 @@ sourcetest: srcck1.exe sqlite3.c fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2177,46 +2184,54 @@ testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ +changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) - $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) + $(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \ + $(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C) + rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) moreclean: clean diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 32d03dbf35..32ef1c5434 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -950,7 +950,7 @@ Replace.exe: sqlite3.def: Replace.exe $(LIBOBJ) echo EXPORTS > sqlite3.def dumpbin /all $(LIBOBJ) \ - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) diff --git a/manifest b/manifest index cf619e924e..a3e5404e7d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sharmless\scompiler\swarnings\sin\sdbfuzz. -D 2017-02-07T21:00:44.802 +C MSVC\smakefile\senhancments\sfor\sseveral\scommand\sline\stools\sand\sthe\ssession\sextensions. +D 2017-02-07T21:09:34.792 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc ba953c8921fc7e18333f61898007206de7e23964 +F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION cddd8d88dc8202afa0ebc96da61fc4acbd1e96a5 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc 3f29e0fc2a78e113250d79a5da375c1d8d8c06b0 +F autoconf/Makefile.msc 3de603bcec5c07462801a20fd7dc1b871400af99 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc @@ -1489,7 +1489,7 @@ F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 F tool/mkkeywordhash.c f7f3b342211ac6a14258b9726d5b97cf4f548f22 -F tool/mkmsvcmin.tcl 2f12f7fa8858bbe61cf81820a2da96c79ed1ca8d +F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1497,7 +1497,7 @@ F tool/mkpragmatab.tcl ebb4bfcd2f8010e0a3934b6118db4b5f2f5edf5c F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e -F tool/mksqlite3h.tcl c006c4e5da57c649b24b689511dcd270dd7b0249 +F tool/mksqlite3h.tcl b9836752c3d08f9fab2dfc0017ca9fd5d90ac863 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db6b39937dd9d6fcd5fcc582c8dd529caad755da -R f5b20543ad5a9de57f328e5cb19dc17f -U drh -Z 556fccfd2bf9ea9a682f73b7b4dda1ae +P 61242267824135a9d5438ec15e3352a2f21dc2fc +R 2db6df3e55ca3fe258c2e43d80c841d4 +U mistachkin +Z ad604e11dbe50606213e8f571c7cab8e diff --git a/manifest.uuid b/manifest.uuid index db9339c4af..4a692b1501 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61242267824135a9d5438ec15e3352a2f21dc2fc \ No newline at end of file +fc1dfe870279531d77ffb18a9ca4b4dcbc7aab8d \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 88e7f9184e..bdd02be1fb 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -83,7 +83,7 @@ Replace.exe: sqlite3.def: Replace.exe $(LIBOBJ) echo EXPORTS > sqlite3.def dumpbin /all $(LIBOBJ) \\ - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\ | sort >> sqlite3.def }]] diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 4f8a1fd497..9d307d1b1e 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -73,7 +73,13 @@ close $in # Set up patterns for recognizing API declarations. # set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} -set declpattern {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$} +set declpattern1 {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern2 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3session_[_a-zA-Z0-9]+)(\(.*)$} + +set declpattern3 \ + {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3changeset_[_a-zA-Z0-9]+)(\(.*)$} # Force the output to use unix line endings, even on Windows. fconfigure stdout -translation lf @@ -121,7 +127,9 @@ foreach file $filelist { if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { set line "SQLITE_API $line" } else { - if {[regexp $declpattern $line all rettype funcname rest]} { + if {[regexp $declpattern1 $line all rettype funcname rest] || \ + [regexp $declpattern2 $line all rettype funcname rest] || \ + [regexp $declpattern3 $line all rettype funcname rest]} { set line SQLITE_API append line " " [string trim $rettype] if {[string index $rettype end] ne "*"} { From ec26ce33281a2ae08852d729f41fd0c1cce09a5e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Feb 2017 21:44:40 +0000 Subject: [PATCH 137/292] When generating the hash on the sqlite_master table in dbselftest, use an ORDER BY clause, since the sqlite_master table is reordered by VACUUM. FossilOrigin-Name: c8bfd99b96608a08f934f46b4e1a4d0f1cc69ea7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/dbselftest.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a3e5404e7d..e98ee3d2a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C MSVC\smakefile\senhancments\sfor\sseveral\scommand\sline\stools\sand\sthe\ssession\sextensions. -D 2017-02-07T21:09:34.792 +C When\sgenerating\sthe\shash\son\sthe\ssqlite_master\stable\sin\sdbselftest,\suse\san\nORDER\sBY\sclause,\ssince\sthe\ssqlite_master\stable\sis\sreordered\sby\sVACUUM. +D 2017-02-07T21:44:40.303 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -636,7 +636,7 @@ F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbselftest.c 4bf86fe04dc2d0d58dabc5e6b7a06fa4ef7827ea +F test/dbselftest.c b2e6cfac59066dbcb7334b66304bb15a5508dd42 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61242267824135a9d5438ec15e3352a2f21dc2fc -R 2db6df3e55ca3fe258c2e43d80c841d4 -U mistachkin -Z ad604e11dbe50606213e8f571c7cab8e +P fc1dfe870279531d77ffb18a9ca4b4dcbc7aab8d +R f274165d340f013add78e0f37722933a +U drh +Z fc92264580c9fd0c7c0e56ba0825054f diff --git a/manifest.uuid b/manifest.uuid index 4a692b1501..b6d3f806f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc1dfe870279531d77ffb18a9ca4b4dcbc7aab8d \ No newline at end of file +c8bfd99b96608a08f934f46b4e1a4d0f1cc69ea7 \ No newline at end of file diff --git a/test/dbselftest.c b/test/dbselftest.c index d8a9c331de..3a238bce16 100644 --- a/test/dbselftest.c +++ b/test/dbselftest.c @@ -568,7 +568,7 @@ static int buildSelftestTable(sqlite3 *db){ ); if( rc ) return 1; tno = 110; - zSql = "SELECT type,name,tbl_name,sql FROM sqlite_master"; + zSql = "SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"; sha1Exec(db, zSql, zHash); rc = runSql(db, "INSERT INTO selftest(tno,op,sql,ans)" From 0356ebd0a55ef92febfb8943bdf3e08542629de8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Feb 2017 12:18:05 +0000 Subject: [PATCH 138/292] Round up the size of all memory allocations to a multiple of 8 bytes when using the system memory allocator. FossilOrigin-Name: c46e06fab4465128ac3364bafef5fa3d016796d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem1.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e98ee3d2a4..ee914ed3f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sgenerating\sthe\shash\son\sthe\ssqlite_master\stable\sin\sdbselftest,\suse\san\nORDER\sBY\sclause,\ssince\sthe\ssqlite_master\stable\sis\sreordered\sby\sVACUUM. -D 2017-02-07T21:44:40.303 +C Round\sup\sthe\ssize\sof\sall\smemory\sallocations\sto\sa\smultiple\sof\s8\sbytes\swhen\nusing\sthe\ssystem\smemory\sallocator. +D 2017-02-08T12:18:05.716 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -362,7 +362,7 @@ F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b +F src/mem1.c d5214ba7ca6315702d4dc19c6a28c28374aabf0f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc1dfe870279531d77ffb18a9ca4b4dcbc7aab8d -R f274165d340f013add78e0f37722933a +P c8bfd99b96608a08f934f46b4e1a4d0f1cc69ea7 +R 18730fab1d81ceb35863e5f55cd414cf U drh -Z fc92264580c9fd0c7c0e56ba0825054f +Z e4ad513632515387e7941b963a76aac0 diff --git a/manifest.uuid b/manifest.uuid index b6d3f806f6..876796e051 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8bfd99b96608a08f934f46b4e1a4d0f1cc69ea7 \ No newline at end of file +c46e06fab4465128ac3364bafef5fa3d016796d0 \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index b960ccfd47..28055190e0 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -125,7 +125,7 @@ static malloc_zone_t* _sqliteZone_; */ static void *sqlite3MemMalloc(int nByte){ #ifdef SQLITE_MALLOCSIZE - void *p = SQLITE_MALLOC( nByte ); + void *p = SQLITE_MALLOC( ROUND8(nByte) ); if( p==0 ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); From 087a29c78be60f5b3c5c9718c3f1843a97781a74 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Feb 2017 16:01:57 +0000 Subject: [PATCH 139/292] Always invoke the xRoundup() method of the memory allocator before calling xMalloc(). FossilOrigin-Name: 77b470b0df73dc5ae5ad2f0170ef7c50558c7c88 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/malloc.c | 16 ++++++++++++---- src/mem1.c | 6 ++++-- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ee914ed3f5..8fb5d8e602 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Round\sup\sthe\ssize\sof\sall\smemory\sallocations\sto\sa\smultiple\sof\s8\sbytes\swhen\nusing\sthe\ssystem\smemory\sallocator. -D 2017-02-08T12:18:05.716 +C Always\sinvoke\sthe\sxRoundup()\smethod\sof\sthe\smemory\sallocator\sbefore\scalling\nxMalloc(). +D 2017-02-08T16:01:57.242 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -360,9 +360,9 @@ F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c fc1b9f445290f2145da48fc08730c26e6082b640 +F src/malloc.c dc2b84cd320e0498ef4209772d66f353df6e4eb2 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c d5214ba7ca6315702d4dc19c6a28c28374aabf0f +F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8bfd99b96608a08f934f46b4e1a4d0f1cc69ea7 -R 18730fab1d81ceb35863e5f55cd414cf +P c46e06fab4465128ac3364bafef5fa3d016796d0 +R 4a0a823e254c393eb2ca1b8aee7648b4 U drh -Z e4ad513632515387e7941b963a76aac0 +Z f41f90df67587790eae7117dc459eaeb diff --git a/manifest.uuid b/manifest.uuid index 876796e051..e1604708ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c46e06fab4465128ac3364bafef5fa3d016796d0 \ No newline at end of file +77b470b0df73dc5ae5ad2f0170ef7c50558c7c88 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 0a3d891e9b..47064aafba 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -219,12 +219,20 @@ static void sqlite3MallocAlarm(int nByte){ */ static void mallocWithAlarm(int n, void **pp){ void *p; - int nFull = 0; + int nFull; assert( sqlite3_mutex_held(mem0.mutex) ); + assert( n>0 ); + + /* In Firefox (circa 2017-02-08), xRoundup is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimization out + ** the following xRoundup() call. */ + nFull = sqlite3GlobalConfig.m.xRoundup(n); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - nFull = sqlite3GlobalConfig.m.xRoundup(n); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -232,11 +240,11 @@ static void mallocWithAlarm(int n, void **pp){ mem0.nearlyFull = 0; } } - p = sqlite3GlobalConfig.m.xMalloc(n); + p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); - p = sqlite3GlobalConfig.m.xMalloc(n); + p = sqlite3GlobalConfig.m.xMalloc(nFull); } #endif if( p ){ diff --git a/src/mem1.c b/src/mem1.c index 28055190e0..efc84c41d7 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -125,7 +125,9 @@ static malloc_zone_t* _sqliteZone_; */ static void *sqlite3MemMalloc(int nByte){ #ifdef SQLITE_MALLOCSIZE - void *p = SQLITE_MALLOC( ROUND8(nByte) ); + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); if( p==0 ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); @@ -134,7 +136,7 @@ static void *sqlite3MemMalloc(int nByte){ #else sqlite3_int64 *p; assert( nByte>0 ); - nByte = ROUND8(nByte); + testcase( ROUND8(nByte)!=nByte ); p = SQLITE_MALLOC( nByte+8 ); if( p ){ p[0] = nByte; From 40b84365e41e5d1183fbcc1df6da67e611830210 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 8 Feb 2017 18:13:46 +0000 Subject: [PATCH 140/292] Typo fixes in comment. No changes to code. FossilOrigin-Name: c09dd5c0befaf5028abfead8114bd74a30ffe5d4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/malloc.c | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8fb5d8e602..cbc9d6ee92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sinvoke\sthe\sxRoundup()\smethod\sof\sthe\smemory\sallocator\sbefore\scalling\nxMalloc(). -D 2017-02-08T16:01:57.242 +C Typo\sfixes\sin\scomment.\s\sNo\schanges\sto\scode. +D 2017-02-08T18:13:46.938 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -360,7 +360,7 @@ F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c dc2b84cd320e0498ef4209772d66f353df6e4eb2 +F src/malloc.c f345c60d093d6c6cf451b99a7344a123b1a70fd9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c46e06fab4465128ac3364bafef5fa3d016796d0 -R 4a0a823e254c393eb2ca1b8aee7648b4 -U drh -Z f41f90df67587790eae7117dc459eaeb +P 77b470b0df73dc5ae5ad2f0170ef7c50558c7c88 +R 52473535d67a86108597185ae6d56e08 +U mistachkin +Z 77244f34774b49c249d3e5a9b6027fee diff --git a/manifest.uuid b/manifest.uuid index e1604708ad..0e6917c22f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77b470b0df73dc5ae5ad2f0170ef7c50558c7c88 \ No newline at end of file +c09dd5c0befaf5028abfead8114bd74a30ffe5d4 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 47064aafba..e7714aa103 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -223,11 +223,11 @@ static void mallocWithAlarm(int n, void **pp){ assert( sqlite3_mutex_held(mem0.mutex) ); assert( n>0 ); - /* In Firefox (circa 2017-02-08), xRoundup is remapped to an internal + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal ** implementation of malloc_good_size(), which must be called in debug ** mode and specifically when the DMD "Dark Matter Detector" is enabled - ** or else a crash results. Hence, do not attempt to optimization out - ** the following xRoundup() call. */ + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); From 18fdde21b8f7b484c1aa32b601b83309876c91d9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Feb 2017 19:10:47 +0000 Subject: [PATCH 141/292] Avoid preparing a SELECT statement each time an UPDATE or DELETE by docid is executed against an fts3 table. FossilOrigin-Name: 9962c10a5c6672bd82b2bf640d878fcdac0b815a --- ext/fts3/fts3.c | 50 +++++++++++++++++++++++++++++++++------------- ext/fts3/fts3Int.h | 2 ++ manifest | 19 ++++++++++-------- manifest.uuid | 2 +- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c12e3d2156..7c931c42d4 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -492,6 +492,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ assert( p->pSegments==0 ); /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } @@ -1680,6 +1681,26 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return SQLITE_OK; } +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -1687,7 +1708,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); @@ -1705,20 +1726,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ ** ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to ** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->pStmt==0 ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; } - *ppStmt = pCsr->pStmt; return rc; } @@ -1730,9 +1754,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); pCsr->isRequireSeek = 0; @@ -3190,7 +3212,7 @@ static int fts3FilterMethod( assert( iIdx==nVal ); /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); @@ -3258,7 +3280,7 @@ static int fts3FilterMethod( rc = SQLITE_NOMEM; } }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0c86c4217e..c3cab9d821 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -230,6 +230,7 @@ struct Fts3Table { ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ char *zReadExprlist; char *zWriteExprlist; @@ -299,6 +300,7 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int iLangid; /* Language being queried for */ diff --git a/manifest b/manifest index cbc9d6ee92..ebac74811d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Typo\sfixes\sin\scomment.\s\sNo\schanges\sto\scode. -D 2017-02-08T18:13:46.938 +C Avoid\spreparing\sa\sSELECT\sstatement\seach\stime\san\sUPDATE\sor\sDELETE\nby\sdocid\sis\sexecuted\sagainst\san\sfts3\stable. +D 2017-02-08T19:10:47.175 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -70,9 +70,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5a8cafedffd101e9946f8909ecf8d34aaa383b4d +F ext/fts3/fts3.c c4d7eecb12de9749851bcab6e5ca616a5803047a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 89d0bd4595a0de384dac78e94b803de12586e8dd +F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 F ext/fts3/fts3_expr.c dfd571a24412779ac01f25c01d888c6ef7b2d0ef F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -1555,7 +1555,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77b470b0df73dc5ae5ad2f0170ef7c50558c7c88 -R 52473535d67a86108597185ae6d56e08 -U mistachkin -Z 77244f34774b49c249d3e5a9b6027fee +P c09dd5c0befaf5028abfead8114bd74a30ffe5d4 +R fbd7643b7e0c4a3c885d2a6aeb5fecfa +T *branch * fts3-seekstmt-cache +T *sym-fts3-seekstmt-cache * +T -sym-trunk * +U dan +Z dc34c96dc2a3bdce03fc410bb45d9619 diff --git a/manifest.uuid b/manifest.uuid index 0e6917c22f..a8184cc911 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c09dd5c0befaf5028abfead8114bd74a30ffe5d4 \ No newline at end of file +9962c10a5c6672bd82b2bf640d878fcdac0b815a \ No newline at end of file From a39284bfa879ef817476854c16ed5a45ec2a1def Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Feb 2017 17:12:22 +0000 Subject: [PATCH 142/292] Cleanup the usage of the SQLITE_DISABLE_INTRINSIC compile-time option. Remove the SQLITE_RUNTIME_BYTEORDER compile-time option. Use -DSQLITE_BYTEORDER=0 instead. Fix a bug in R-Tree that occurs when compiling on a known little-endian machine without the use of intrinsic byteswapping functions. FossilOrigin-Name: 798fb9d70d2e5f95e64237b04d6692360133381a --- ext/rtree/rtree.c | 23 +++++++++-------------- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/btreeInt.h | 6 ++---- src/mutex_w32.c | 3 +-- src/sqliteInt.h | 48 +++++++++++++++++++++++++++++------------------ src/util.c | 21 +++++++-------------- 7 files changed, 61 insertions(+), 64 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 1e3e65ecc9..205939ddc6 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -370,7 +370,7 @@ struct RtreeMatchArg { /* What version of GCC is being used. 0 means GCC is not being used */ #ifndef GCC_VERSION -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) #else # define GCC_VERSION 0 @@ -379,7 +379,7 @@ struct RtreeMatchArg { /* What version of CLANG is being used. 0 means CLANG is not being used */ #ifndef CLANG_VERSION -#if defined(__clang__) && !defined(_WIN32) +#if defined(__clang__) && !defined(_WIN32) && !defined(SQLITE_DISABLE_INTRINSIC) # define CLANG_VERSION \ (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) #else @@ -404,13 +404,12 @@ struct RtreeMatchArg { ** at run-time. */ #ifndef SQLITE_BYTEORDER -#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) +#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) # define SQLITE_BYTEORDER 1234 -#elif (defined(sparc) || defined(__ppc__)) \ - && !defined(SQLITE_RUNTIME_BYTEORDER) +#elif defined(sparc) || defined(__ppc__) # define SQLITE_BYTEORDER 4321 #else # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ @@ -420,7 +419,7 @@ struct RtreeMatchArg { /* What version of MSVC is being used. 0 means MSVC is not being used */ #ifndef MSVC_VERSION -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) # define MSVC_VERSION _MSC_VER #else # define MSVC_VERSION 0 @@ -440,9 +439,6 @@ static void readCoord(u8 *p, RtreeCoord *pCoord){ pCoord->u = _byteswap_ulong(*(u32*)p); #elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) pCoord->u = __builtin_bswap32(*(u32*)p); -#elif SQLITE_BYTEORDER==1234 - pCoord->u = ((pCoord->u>>24)&0xff)|((pCoord->u>>8)&0xff00)| - ((pCoord->u&0xff)<<24)|((pCoord->u&0xff00)<<8); #elif SQLITE_BYTEORDER==4321 pCoord->u = *(u32*)p; #else @@ -486,10 +482,9 @@ static i64 readInt64(u8 *p){ ** 64 bit integer. The value returned is the number of bytes written ** to the argument buffer (always 2, 4 and 8 respectively). */ -static int writeInt16(u8 *p, int i){ +static void writeInt16(u8 *p, int i){ p[0] = (i>> 8)&0xFF; p[1] = (i>> 0)&0xFF; - return 2; } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; diff --git a/manifest b/manifest index 5f267df71a..3dd0a8d9c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\spreparing\sa\sSELECT\sstatement\seach\stime\san\sUPDATE\sor\sDELETE\sby\sdocid\sis\nexecuted\sagainst\san\sfts3\stable. -D 2017-02-08T19:12:36.968 +C Cleanup\sthe\susage\sof\sthe\sSQLITE_DISABLE_INTRINSIC\scompile-time\soption.\nRemove\sthe\sSQLITE_RUNTIME_BYTEORDER\scompile-time\soption.\s\sUse\n-DSQLITE_BYTEORDER=0\sinstead.\s\sFix\sa\sbug\sin\sR-Tree\sthat\soccurs\swhen\scompiling\non\sa\sknown\slittle-endian\smachine\swithout\sthe\suse\sof\sintrinsic\sbyteswapping\nfunctions. +D 2017-02-09T17:12:22.122 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d92be99e3a662e1934658201f0fc7ad51b44325a +F ext/rtree/rtree.c 358796a385de74e40ed66c103df77ba3c2e98fab F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -339,7 +339,7 @@ F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 9fe65ab418d99e80289f024016b5e5e74c3059dd F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac -F src/btreeInt.h 10c4b77c2fb399580babbcc7cf652ac10dba796e +F src/btreeInt.h 429bbfebae05ab3beb1c4508ba94c292036c660f F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -372,7 +372,7 @@ F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 +F src/mutex_w32.c 00bbf37d80fb763a8159b83cc3cbde0f91d37f7b F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 @@ -399,7 +399,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 3724c48e82605b471bbf7e41109b5850c6e89f31 +F src/sqliteInt.h 4dc66ec1948765f151899333f7a55267d807d099 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -459,7 +459,7 @@ F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c a88b0466fddf445ce752226d4698ca3faada620a +F src/util.c 3d2ce209a89b95cf35bffa16440f57368576e2ee F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 F src/vdbe.c e7b1e860140f1d1803c6f4176b1e8f3801f3a290 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c09dd5c0befaf5028abfead8114bd74a30ffe5d4 9962c10a5c6672bd82b2bf640d878fcdac0b815a -R fbd7643b7e0c4a3c885d2a6aeb5fecfa -U dan -Z a7e2195607498eebba62ce2716ddc0de +P 1afec5758b624e6a066d4e7ef50695095e9d7ff1 +R d99a1e96a3817ca7543fc882a1f70881 +U drh +Z 871c0b5ab0829dba2d37da91622734c8 diff --git a/manifest.uuid b/manifest.uuid index 6a403cde77..afdafcf9aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1afec5758b624e6a066d4e7ef50695095e9d7ff1 \ No newline at end of file +798fb9d70d2e5f95e64237b04d6692360133381a \ No newline at end of file diff --git a/src/btreeInt.h b/src/btreeInt.h index fc2182b634..0d79edd639 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -692,11 +692,9 @@ struct IntegrityCk { */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSCV_VERSION>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9570bdc0bf..e0bdf9e43b 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -87,8 +87,7 @@ void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ - defined(_MSC_VER) && _MSC_VER>=1300 +#elif MSCV_VERSION>=1300 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3faf9d3982..f9768895ae 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -103,20 +103,28 @@ # define _LARGEFILE_SOURCE 1 #endif -/* What version of GCC is being used. 0 means GCC is not being used */ -#ifdef __GNUC__ +/* The GCC_VERSION, CLANG_VERSION, and MSVC_VERSION macros are used to +** conditionally include optimizations for each of these compilers. A +** value of 0 means that compiler is not being used. The +** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific +** optimizations, and hence set all compiler macros to 0 +*/ +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) #else # define GCC_VERSION 0 #endif - -/* What version of CLANG is being used. 0 means CLANG is not being used */ -#if defined(__clang__) && !defined(_WIN32) +#if defined(__clang__) && !defined(_WIN32) && !defined(SQLITE_DISABLE_INTRINSIC) # define CLANG_VERSION \ (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) #else # define CLANG_VERSION 0 #endif +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif /* Needed for various definitions... */ #if defined(__GNUC__) && !defined(_GNU_SOURCE) @@ -247,6 +255,7 @@ # include # pragma intrinsic(_byteswap_ushort) # pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) # pragma intrinsic(_ReadWriteBarrier) # else # include @@ -787,32 +796,35 @@ typedef INT16_TYPE LogEst; ** ** For best performance, an attempt is made to guess at the byte-order ** using C-preprocessor macros. If that is unsuccessful, or if -** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined ** at run-time. */ -#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +#ifndef SQLITE_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 1234 -# define SQLITE_BIGENDIAN 0 -# define SQLITE_LITTLEENDIAN 1 -# define SQLITE_UTF16NATIVE SQLITE_UTF16LE + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif -#if (defined(sparc) || defined(__ppc__)) \ - && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 4321 +#if SQLITE_BYTEORDER==4321 # define SQLITE_BIGENDIAN 1 # define SQLITE_LITTLEENDIAN 0 # define SQLITE_UTF16NATIVE SQLITE_UTF16BE -#endif -#if !defined(SQLITE_BYTEORDER) +#elif SQLITE_BYTEORDER==1234 +# define SQLITE_BIGENDIAN 0 +# define SQLITE_LITTLEENDIAN 1 +# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#else # ifdef SQLITE_AMALGAMATION const int sqlite3one = 1; # else extern const int sqlite3one; # endif -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) # define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) diff --git a/src/util.c b/src/util.c index ca14ab8526..c6d2bae3a7 100644 --- a/src/util.c +++ b/src/util.c @@ -1140,13 +1140,11 @@ u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x; memcpy(&x,p,4); return _byteswap_ulong(x); @@ -1158,12 +1156,10 @@ u32 sqlite3Get4byte(const u8 *p){ void sqlite3Put4byte(unsigned char *p, u32 v){ #if SQLITE_BYTEORDER==4321 memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) u32 x = __builtin_bswap32(v); memcpy(p,&x,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x = _byteswap_ulong(v); memcpy(p,&x,4); #else @@ -1279,8 +1275,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ ** overflow, leave *pA unchanged and return 1. */ int sqlite3AddInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 return __builtin_add_overflow(*pA, iB, pA); #else i64 iA = *pA; @@ -1300,8 +1295,7 @@ int sqlite3AddInt64(i64 *pA, i64 iB){ #endif } int sqlite3SubInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 return __builtin_sub_overflow(*pA, iB, pA); #else testcase( iB==SMALLEST_INT64+1 ); @@ -1316,8 +1310,7 @@ int sqlite3SubInt64(i64 *pA, i64 iB){ #endif } int sqlite3MulInt64(i64 *pA, i64 iB){ -#if !defined(SQLITE_DISABLE_INTRINSIC) \ - && (GCC_VERSION>=5004000 || CLANG_VERSION>=4000000) +#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 return __builtin_mul_overflow(*pA, iB, pA); #else i64 iA = *pA; From 2c338a9d9ad1f05269a4d44e5bf9fdaeac51a95c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Feb 2017 19:38:36 +0000 Subject: [PATCH 143/292] Add the "," flag to printf(). FossilOrigin-Name: 064445b12f99f76e9a12957be97edd520ab3ae27 --- manifest | 14 +++---- manifest.uuid | 2 +- src/printf.c | 97 +++++++++++++++++++++++++++-------------------- test/printf2.test | 31 +++++++++++++++ 4 files changed, 94 insertions(+), 50 deletions(-) diff --git a/manifest b/manifest index 3dd0a8d9c4..53d7e16ed6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanup\sthe\susage\sof\sthe\sSQLITE_DISABLE_INTRINSIC\scompile-time\soption.\nRemove\sthe\sSQLITE_RUNTIME_BYTEORDER\scompile-time\soption.\s\sUse\n-DSQLITE_BYTEORDER=0\sinstead.\s\sFix\sa\sbug\sin\sR-Tree\sthat\soccurs\swhen\scompiling\non\sa\sknown\slittle-endian\smachine\swithout\sthe\suse\sof\sintrinsic\sbyteswapping\nfunctions. -D 2017-02-09T17:12:22.122 +C Add\sthe\s","\sflag\sto\sprintf(). +D 2017-02-10T19:38:36.506 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -390,7 +390,7 @@ F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a -F src/printf.c ff10a9b9902cd2afe5f655f3013c6307d969b1fd +F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1021,7 +1021,7 @@ F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 +F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1afec5758b624e6a066d4e7ef50695095e9d7ff1 -R d99a1e96a3817ca7543fc882a1f70881 +P 798fb9d70d2e5f95e64237b04d6692360133381a +R c3dc7f748658995c87bb25e10706fab3 U drh -Z 871c0b5ab0829dba2d37da91622734c8 +Z 3add84968b9ef1dfcf3d7b3eaefd30ad diff --git a/manifest.uuid b/manifest.uuid index afdafcf9aa..8090b2db36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -798fb9d70d2e5f95e64237b04d6692360133381a \ No newline at end of file +064445b12f99f76e9a12957be97edd520ab3ae27 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index ede86f1208..241338b266 100644 --- a/src/printf.c +++ b/src/printf.c @@ -15,7 +15,7 @@ ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ +#define etRADIX 0 /* non-decimal integer types. %x %o */ #define etFLOAT 1 /* Floating point. %f */ #define etEXP 2 /* Exponentional notation. %e and %E */ #define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ @@ -33,8 +33,9 @@ #define etPOINTER 13 /* The %p conversion */ #define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ #define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 16 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -58,8 +59,8 @@ typedef struct et_info { /* Information about each format field */ /* ** Allowed values for et_info.flags */ -#define FLAG_SIGNED 1 /* True if the value to convert is signed */ -#define FLAG_STRING 4 /* Allow infinity precision */ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ /* @@ -69,7 +70,7 @@ typedef struct et_info { /* Information about each format field */ static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; static const et_info fmtinfo[] = { - { 'd', 10, 1, etRADIX, 0, 0 }, + { 'd', 10, 1, etDECIMAL, 0, 0 }, { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, { 'z', 0, 4, etDYNSTRING, 0, 0 }, @@ -78,7 +79,7 @@ static const et_info fmtinfo[] = { { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, { 'c', 0, 0, etCHARX, 0, 0 }, { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etRADIX, 0, 0 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, { 'x', 16, 0, etRADIX, 16, 1 }, { 'X', 16, 0, etRADIX, 0, 4 }, #ifndef SQLITE_OMIT_FLOATING_POINT @@ -87,7 +88,7 @@ static const et_info fmtinfo[] = { { 'E', 0, 1, etEXP, 14, 0 }, { 'G', 0, 1, etGENERIC, 14, 0 }, #endif - { 'i', 10, 1, etRADIX, 0, 0 }, + { 'i', 10, 1, etDECIMAL, 0, 0 }, { 'n', 0, 0, etSIZE, 0, 0 }, { '%', 0, 0, etPERCENT, 0, 0 }, { 'p', 16, 0, etPOINTER, 0, 1 }, @@ -179,14 +180,13 @@ void sqlite3VXPrintf( int idx; /* A general purpose loop counter */ int width; /* Width of the current field */ etByte flag_leftjustify; /* True if "-" flag is present */ - etByte flag_plussign; /* True if "+" flag is present */ - etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ etByte flag_alternateform; /* True if "#" flag is present */ etByte flag_altform2; /* True if "!" flag is present */ etByte flag_zeropad; /* True if field width constant starts with zero */ - etByte flag_long; /* True if "l" flag is present */ - etByte flag_longlong; /* True if the "ll" flag is present */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ etByte done; /* Loop termination flag */ + etByte cThousand; /* Thousands separator for %d and %u */ etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ @@ -229,17 +229,18 @@ void sqlite3VXPrintf( break; } /* Find out what flags are present */ - flag_leftjustify = flag_plussign = flag_blanksign = + flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; do{ switch( c ){ case '-': flag_leftjustify = 1; break; - case '+': flag_plussign = 1; break; - case ' ': flag_blanksign = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; case '#': flag_alternateform = 1; break; case '!': flag_altform2 = 1; break; case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; default: done = 1; break; } }while( !done && (c=(*++fmt))!=0 ); @@ -309,13 +310,11 @@ void sqlite3VXPrintf( flag_long = 1; c = *++fmt; if( c=='l' ){ - flag_longlong = 1; + flag_long = 2; c = *++fmt; - }else{ - flag_longlong = 0; } }else{ - flag_long = flag_longlong = 0; + flag_long = 0; } /* Fetch the info entry for the field */ infop = &fmtinfo[0]; @@ -333,15 +332,11 @@ void sqlite3VXPrintf( ** ** flag_alternateform TRUE if a '#' is present. ** flag_altform2 TRUE if a '!' is present. - ** flag_plussign TRUE if a '+' is present. + ** flag_prefix '+' or ' ' or zero ** flag_leftjustify TRUE if a '-' is present or if the ** field width was negative. ** flag_zeropad TRUE if the width began with 0. - ** flag_long TRUE if the letter 'l' (ell) prefixed - ** the conversion character. - ** flag_longlong TRUE if the letter 'll' (ell ell) prefixed - ** the conversion character. - ** flag_blanksign TRUE if a ' ' is present. + ** flag_long 1 for "l", 2 for "ll" ** width The specified field width. This is ** always non-negative. Zero is the default. ** precision The specified precision. The default @@ -351,19 +346,24 @@ void sqlite3VXPrintf( */ switch( xtype ){ case etPOINTER: - flag_longlong = sizeof(char*)==sizeof(i64); - flag_long = sizeof(char*)==sizeof(long int); + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; /* Fall through into the next case */ case etORDINAL: - case etRADIX: + case etRADIX: + cThousand = 0; + /* Fall through into the next case */ + case etDECIMAL: if( infop->flags & FLAG_SIGNED ){ i64 v; if( bArgList ){ v = getIntArg(pArgList); - }else if( flag_longlong ){ - v = va_arg(ap,i64); }else if( flag_long ){ - v = va_arg(ap,long int); + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } }else{ v = va_arg(ap,int); } @@ -376,17 +376,17 @@ void sqlite3VXPrintf( prefix = '-'; }else{ longvalue = v; - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } }else{ if( bArgList ){ longvalue = (u64)getIntArg(pArgList); - }else if( flag_longlong ){ - longvalue = va_arg(ap,u64); }else if( flag_long ){ - longvalue = va_arg(ap,unsigned long int); + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } }else{ longvalue = va_arg(ap,unsigned int); } @@ -396,11 +396,11 @@ void sqlite3VXPrintf( if( flag_zeropad && precision0 ); } length = (int)(&zOut[nOut-1]-bufpt); - for(idx=precision-length; idx>0; idx--){ + while( precision>length ){ *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } } if( prefix ) *(--bufpt) = prefix; /* Add sign */ if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ @@ -454,9 +469,7 @@ void sqlite3VXPrintf( realvalue = -realvalue; prefix = '-'; }else{ - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); diff --git a/test/printf2.test b/test/printf2.test index fb031bd68f..d30966d167 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -116,6 +116,37 @@ do_execsql_test printf2-3.5 { SELECT printf('|%7.8c|%-7.8c|','*','*'); } {|********|********|} +# The "," separator +do_execsql_test printf2-4.1 { + SELECT printf('|%,d|%,d|',0,-1); +} {|0|-1|} +do_execsql_test printf2-4.2 { + SELECT printf('|%,d|%,d|',12,-12); +} {|12|-12|} +do_execsql_test printf2-4.3 { + SELECT printf('|%,d|%,d|',123,-123); +} {|123|-123|} +do_execsql_test printf2-4.4 { + SELECT printf('|%,d|%,d|',1234,-1234); +} {|1,234|-1,234|} +do_execsql_test printf2-4.5 { + SELECT printf('|%,d|%,d|',12345,-12345); +} {|12,345|-12,345|} +do_execsql_test printf2-4.6 { + SELECT printf('|%,d|%,d|',123456,-123456); +} {|123,456|-123,456|} +do_execsql_test printf2-4.7 { + SELECT printf('|%,d|%,d|',1234567,-1234567); +} {|1,234,567|-1,234,567|} +do_execsql_test printf2-4.8 { + SELECT printf('|%,d|%,d|',12345678,-12345678); +} {|12,345,678|-12,345,678|} +do_execsql_test printf2-4.9 { + SELECT printf('|%,d|%,d|',123456789,-123456789); +} {|123,456,789|-123,456,789|} +do_execsql_test printf2-4.10 { + SELECT printf('|%,d|%,d|',1234567890,-1234567890); +} {|1,234,567,890|-1,234,567,890|} From 1c84bd478776a79b7ba5f8c73743a873ee7c4994 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Feb 2017 21:37:57 +0000 Subject: [PATCH 144/292] Enhance the LIKE optimization so that it works for arbitrary expressions on the LHS as long as the pattern on the RHS does not begin with a digit or a minus sign. FossilOrigin-Name: 158290c0abafde67ee3f2363f0b6646887841df3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/whereexpr.c | 26 +++++++++++++++++--------- test/vtab1.test | 28 ++++++++++++++-------------- test/vtabH.test | 8 ++++---- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index 53d7e16ed6..9d7931aab5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s","\sflag\sto\sprintf(). -D 2017-02-10T19:38:36.506 +C Enhance\sthe\sLIKE\soptimization\sso\sthat\sit\sworks\sfor\sarbitrary\sexpressions\son\nthe\sLHS\sas\slong\sas\sthe\spattern\son\sthe\sRHS\sdoes\snot\sbegin\swith\sa\sdigit\sor\na\sminus\ssign. +D 2017-02-10T21:37:57.808 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -478,7 +478,7 @@ F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c bc71775e23d23334e8f449aa31012d692dc09cb2 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c 99a8ced164c75edf41b3a865a75381c9adb38b28 -F src/whereexpr.c 35ad025389a632a3987a35617c878be3b3d70dc6 +F src/whereexpr.c 980109826ba02750421c3fa7ab0ecabbac0a639d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1374,7 +1374,7 @@ F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab -F test/vtab1.test 7c4b81abd88361ada9cbe414c459efca26be6bda +F test/vtab1.test ed4a576231d8a36e70fd18e2b79b621b31e7f22a F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 8e73ed268f3d596bc3590f45fc948fb40f28e9c3 @@ -1389,7 +1389,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test a2912cd3ea3386aecc3cb74ebfdfcc4e6d4b7dd3 +F test/vtabH.test 5f9253eb9e41ba9fe94f4aa3e9230191893d7764 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 798fb9d70d2e5f95e64237b04d6692360133381a -R c3dc7f748658995c87bb25e10706fab3 +P 064445b12f99f76e9a12957be97edd520ab3ae27 +R db970521e4fc7396c9c380bbcef0e8ad U drh -Z 3add84968b9ef1dfcf3d7b3eaefd30ad +Z 2c915b2d7e6621ed4f4d2fbc11d0cc6c diff --git a/manifest.uuid b/manifest.uuid index 8090b2db36..0e09e9b731 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -064445b12f99f76e9a12957be97edd520ab3ae27 \ No newline at end of file +158290c0abafde67ee3f2363f0b6646887841df3 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 826d329b7f..f511452e52 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -213,15 +213,6 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; @@ -238,6 +229,23 @@ static int isLikeOrGlob( z = pRight->u.zToken; } if( z ){ + + /* If the RHS begins with a digit or a minus sign, then the LHS must + ** be an ordinary column (not a virtual table column) with TEXT affinity. + ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false + ** even though "lhs LIKE rhs" is true. But if the RHS does not start + ** with a digit or '-', then "lhs LIKE rhs" will always be false if + ** the LHS is numeric and so the optimization still works. + */ + if( sqlite3Isdigit(z[0]) || z[0]=='-' ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ + ){ + sqlite3ValueFree(pVal); + return 0; + } + } cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; diff --git a/test/vtab1.test b/test/vtab1.test index 6b6a0e2683..59ab959746 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -1295,25 +1295,25 @@ do_execsql_test 18.1.0 { CREATE INDEX i6 ON t6(b, a); INSERT INTO t6 VALUES(1, 'Peter'); INSERT INTO t6 VALUES(2, 'Andrew'); - INSERT INTO t6 VALUES(3, 'James'); - INSERT INTO t6 VALUES(4, 'John'); + INSERT INTO t6 VALUES(3, '8James'); + INSERT INTO t6 VALUES(4, '8John'); INSERT INTO t6 VALUES(5, 'Phillip'); INSERT INTO t6 VALUES(6, 'Bartholomew'); CREATE VIRTUAL TABLE e6 USING echo(t6); } foreach {tn sql res filter} { - 1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} James} + 1.1 "SELECT a FROM e6 WHERE b>'8James'" {4 2 6 1 5} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} 8James} - 1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} J K} + 1.2 "SELECT a FROM e6 WHERE b>='8' AND b<'9'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} 8 9} - 1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%} + 1.3 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%} - 1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%} + 1.4 "SELECT a FROM e6 WHERE b LIKE '8j%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%} } { set echo_module {} do_execsql_test 18.$tn.1 $sql $res @@ -1322,11 +1322,11 @@ foreach {tn sql res filter} { do_execsql_test 18.2.0 { PRAGMA case_sensitive_like = ON } foreach {tn sql res filter} { - 2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%} + 2.1 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%} - 2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {} - {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%} + 2.2 "SELECT a FROM e6 WHERE b LIKE '8j%'" {} + {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%} } { set echo_module {} do_execsql_test 18.$tn.1 $sql $res diff --git a/test/vtabH.test b/test/vtabH.test index 3ce457ff0b..c5684ff516 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -31,14 +31,14 @@ do_execsql_test 1.0 { } foreach {tn sql expect} { - 1 "SELECT * FROM e6 WHERE b LIKE 'abc'" { + 1 "SELECT * FROM e6 WHERE b LIKE '8abc'" { xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b like ?} - xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} abc + xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8abc } - 2 "SELECT * FROM e6 WHERE b GLOB 'abc'" { + 2 "SELECT * FROM e6 WHERE b GLOB '8abc'" { xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b glob ?} - xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} abc + xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} 8abc } } { do_test 1.$tn { From 653a5f4e7141ba137721d3ffb79ec89334195962 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Feb 2017 21:40:04 +0000 Subject: [PATCH 145/292] Bump the version number up to 3.18.0. FossilOrigin-Name: 7520c238558346d421e3c24cb7d17a54d1aa56b2 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 3f67e25cea..c5b45eb7b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.17.0 +3.18.0 diff --git a/configure b/configure index be7c89143b..628e23f2d0 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.17.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.18.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.17.0' -PACKAGE_STRING='sqlite 3.17.0' +PACKAGE_VERSION='3.18.0' +PACKAGE_STRING='sqlite 3.18.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1463,7 +1463,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.17.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.18.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1528,7 +1528,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.17.0:";; + short | recursive ) echo "Configuration of sqlite 3.18.0:";; esac cat <<\_ACEOF @@ -1652,7 +1652,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.17.0 +sqlite configure 3.18.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2071,7 +2071,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.17.0, which was +It was created by sqlite $as_me 3.18.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12151,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.17.0, which was +This file was extended by sqlite $as_me 3.18.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12217,7 +12217,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.17.0 +sqlite config.status 3.18.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index 9d7931aab5..67413c58d2 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Enhance\sthe\sLIKE\soptimization\sso\sthat\sit\sworks\sfor\sarbitrary\sexpressions\son\nthe\sLHS\sas\slong\sas\sthe\spattern\son\sthe\sRHS\sdoes\snot\sbegin\swith\sa\sdigit\sor\na\sminus\ssign. -D 2017-02-10T21:37:57.808 +C Bump\sthe\sversion\snumber\sup\sto\s3.18.0. +D 2017-02-10T21:40:04.693 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 -F VERSION cddd8d88dc8202afa0ebc96da61fc4acbd1e96a5 +F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -30,7 +30,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure fcfa56d5d66668c9263dc007724f62acff06dc83 x +F configure b15d030a8ae2e672d76c97b6667dacb98934a467 x F configure.ac 605173e829ab64514ed89f9b53d0da1739d7b0a0 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 064445b12f99f76e9a12957be97edd520ab3ae27 -R db970521e4fc7396c9c380bbcef0e8ad +P 158290c0abafde67ee3f2363f0b6646887841df3 +R 48ad943b0cb23a44970ddc9873b9da0a U drh -Z 2c915b2d7e6621ed4f4d2fbc11d0cc6c +Z faf6e7fd0bb1d717431a908ec9819058 diff --git a/manifest.uuid b/manifest.uuid index 0e09e9b731..461c4b68e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -158290c0abafde67ee3f2363f0b6646887841df3 \ No newline at end of file +7520c238558346d421e3c24cb7d17a54d1aa56b2 \ No newline at end of file From 13ac46eea2effd6e6f6033e062b760665bf4b7d3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2017 13:51:23 +0000 Subject: [PATCH 146/292] Ensure that indexed expressions with collating sequences are handled correctly. Proposed fix for ticket [eb703ba7b50c1a5]. FossilOrigin-Name: 9689d04b8250139e32078b2aa9748edcc6231bcd --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 2 +- src/where.c | 1 + test/indexexpr1.test | 10 ++++++++++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 67413c58d2..c25697f824 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bump\sthe\sversion\snumber\sup\sto\s3.18.0. -D 2017-02-10T21:40:04.693 +C Ensure\sthat\sindexed\sexpressions\swith\scollating\ssequences\sare\shandled\ncorrectly.\s\sProposed\sfix\sfor\sticket\s[eb703ba7b50c1a5]. +D 2017-02-11T13:51:23.762 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -347,7 +347,7 @@ F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c d29114e9b709eaeaaa18553a5bbe60a19302aeef +F src/expr.c c218ec8cfc12b2c5b354660ce7285dfaf43305e9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -475,7 +475,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c bc71775e23d23334e8f449aa31012d692dc09cb2 +F src/where.c b0d81c6f24ea6aebb249c92c5d78ec8ab1452523 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c 99a8ced164c75edf41b3a865a75381c9adb38b28 F src/whereexpr.c 980109826ba02750421c3fa7ab0ecabbac0a639d @@ -867,7 +867,7 @@ F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7 F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 -F test/indexexpr1.test 7d243fac508b4a99fb900ffe34eb488312cfce84 +F test/indexexpr1.test 038b3befa74e5a75126b6e9dd2ae5df61c1c7cf7 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 158290c0abafde67ee3f2363f0b6646887841df3 -R 48ad943b0cb23a44970ddc9873b9da0a +P 7520c238558346d421e3c24cb7d17a54d1aa56b2 +R 89fd81518c2edceda83a282bdfdfc51e U drh -Z faf6e7fd0bb1d717431a908ec9819058 +Z 86c7ba615f1277d92ec3bd479587e4c2 diff --git a/manifest.uuid b/manifest.uuid index 461c4b68e4..9191af5c88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7520c238558346d421e3c24cb7d17a54d1aa56b2 \ No newline at end of file +9689d04b8250139e32078b2aa9748edcc6231bcd \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f1e8667b39..ff4f1b5ade 100644 --- a/src/expr.c +++ b/src/expr.c @@ -231,7 +231,7 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( NEVER(aff==0) ){ + }else if( aff==0 ){ aff = SQLITE_AFF_BLOB; } return aff; diff --git a/src/where.c b/src/where.c index 4951f6a1b4..80dfa20ed1 100644 --- a/src/where.c +++ b/src/where.c @@ -308,6 +308,7 @@ static WhereTerm *whereScanInit( iColumn = pIdx->aiColumn[j]; if( iColumn==XN_EXPR ){ pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + pScan->zCollName = pIdx->azColl[j]; }else if( iColumn==pIdx->pTable->iPKey ){ iColumn = XN_ROWID; }else if( iColumn>=0 ){ diff --git a/test/indexexpr1.test b/test/indexexpr1.test index cd72430d91..2b375a5855 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -370,4 +370,14 @@ do_execsql_test indexexpr1-1200.4 { 0 0 0 2 0 4 2 0 2 2 4 0 } +# Ticket https://www.sqlite.org/src/tktview/eb703ba7b50c1a +# Incorrect result using an index on an expression with a collating function +# +do_execsql_test indexexpr1-1300.1 { + CREATE TABLE t1300(a INTEGER PRIMARY KEY, b); + INSERT INTO t1300 VALUES(1,'coffee'),(2,'COFFEE'),(3,'stress'),(4,'STRESS'); + CREATE INDEX t1300bexpr ON t1300( substr(b,4) ); + SELECT a FROM t1300 WHERE substr(b,4)='ess' COLLATE nocase ORDER BY +a; +} {3 4} + finish_test From f9463dfbcf8069bb1df1dd3407bc6e2837f9cee6 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2017 14:59:58 +0000 Subject: [PATCH 147/292] Fix indexes on expressions so that they can be actually used with a COLLATE clause. FossilOrigin-Name: e464b919f76520b45bb58983c6702db59d820ee4 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 11 +++++++++++ src/sqliteInt.h | 1 + src/where.c | 3 ++- src/whereexpr.c | 2 +- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c25697f824..aff48f9780 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sindexed\sexpressions\swith\scollating\ssequences\sare\shandled\ncorrectly.\s\sProposed\sfix\sfor\sticket\s[eb703ba7b50c1a5]. -D 2017-02-11T13:51:23.762 +C Fix\sindexes\son\sexpressions\sso\sthat\sthey\scan\sbe\sactually\sused\swith\na\sCOLLATE\sclause. +D 2017-02-11T14:59:58.753 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -347,7 +347,7 @@ F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c c218ec8cfc12b2c5b354660ce7285dfaf43305e9 +F src/expr.c 4748f51a680ded80332aca51cc51548c60758fe9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -399,7 +399,7 @@ F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 4dc66ec1948765f151899333f7a55267d807d099 +F src/sqliteInt.h c3f878dcbe947938f9e0984644f1902dd9051094 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -475,10 +475,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c b0d81c6f24ea6aebb249c92c5d78ec8ab1452523 +F src/where.c 6397fab50fdbf9bde76c574ce07b3b776eb28b34 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d F src/wherecode.c 99a8ced164c75edf41b3a865a75381c9adb38b28 -F src/whereexpr.c 980109826ba02750421c3fa7ab0ecabbac0a639d +F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7520c238558346d421e3c24cb7d17a54d1aa56b2 -R 89fd81518c2edceda83a282bdfdfc51e +P 9689d04b8250139e32078b2aa9748edcc6231bcd +R e95d56d566e83b0838849dba73bfcd3a U drh -Z 86c7ba615f1277d92ec3bd479587e4c2 +Z 0aabcf0e8948d008893cd3f00b6802af diff --git a/manifest.uuid b/manifest.uuid index 9191af5c88..85764546dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9689d04b8250139e32078b2aa9748edcc6231bcd \ No newline at end of file +e464b919f76520b45bb58983c6702db59d820ee4 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ff4f1b5ade..6a1a9288a1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4680,6 +4680,17 @@ int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ return 0; } +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ + return sqlite3ExprCompare( + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), + iTab); +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f9768895ae..23b4dd9460 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3762,6 +3762,7 @@ void sqlite3Vacuum(Parse*,Token*); int sqlite3RunVacuum(char**, sqlite3*, int); char *sqlite3NameFromToken(sqlite3*, Token*); int sqlite3ExprCompare(Expr*, Expr*, int); +int sqlite3ExprCompareSkip(Expr*, Expr*, int); int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Expr*, Expr*, int); void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); diff --git a/src/where.c b/src/where.c index 80dfa20ed1..4c533aa493 100644 --- a/src/where.c +++ b/src/where.c @@ -209,7 +209,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn && (iColumn!=XN_EXPR - || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, + pScan->pIdxExpr,iCur)==0) && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 diff --git a/src/whereexpr.c b/src/whereexpr.c index f511452e52..248b5349db 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -874,7 +874,7 @@ static int exprMightBeIndexed( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ *piCur = iCur; *piColumn = XN_EXPR; return 1; From ac279be98ebbe5cce899c549f12d16204d460a67 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2017 13:20:02 +0000 Subject: [PATCH 148/292] Avoid a duplication #define in FTS5 FossilOrigin-Name: c447441cff1884d6fe5f0a76d64b3e7d908584a1 --- ext/fts5/fts5Int.h | 4 +++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9ef338e821..d350856573 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -30,7 +30,9 @@ typedef short i16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif #define testcase(x) #define ALWAYS(x) 1 diff --git a/manifest b/manifest index aff48f9780..b5ae3ca250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sindexes\son\sexpressions\sso\sthat\sthey\scan\sbe\sactually\sused\swith\na\sCOLLATE\sclause. -D 2017-02-11T14:59:58.753 +C Avoid\sa\sduplication\s#define\sin\sFTS5 +D 2017-02-13T13:20:02.419 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -98,7 +98,7 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 -F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f +F ext/fts5/fts5Int.h c629b24d2b92b99596f3b8e82289fddca06df6e1 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9689d04b8250139e32078b2aa9748edcc6231bcd -R e95d56d566e83b0838849dba73bfcd3a +P e464b919f76520b45bb58983c6702db59d820ee4 +R 8c09ada0ea1970f0e52c7f9e3e6704e9 U drh -Z 0aabcf0e8948d008893cd3f00b6802af +Z 6a635e248bfa56654adaa33accae463e diff --git a/manifest.uuid b/manifest.uuid index 85764546dc..a7d1b63450 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e464b919f76520b45bb58983c6702db59d820ee4 \ No newline at end of file +c447441cff1884d6fe5f0a76d64b3e7d908584a1 \ No newline at end of file From 30a5831c45f5c51840fb8017bc1024e1bb058b53 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2017 13:26:33 +0000 Subject: [PATCH 149/292] Fix typos in using the MSVC_VERSION macro. FossilOrigin-Name: f3b65926b1f439adb95e3bbce8e58785b8cf8427 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/btreeInt.h | 2 +- src/mutex_w32.c | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b5ae3ca250..fbd6c3cbf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sduplication\s#define\sin\sFTS5 -D 2017-02-13T13:20:02.419 +C Fix\stypos\sin\susing\sthe\sMSVC_VERSION\smacro. +D 2017-02-13T13:26:33.213 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -339,7 +339,7 @@ F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 9fe65ab418d99e80289f024016b5e5e74c3059dd F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac -F src/btreeInt.h 429bbfebae05ab3beb1c4508ba94c292036c660f +F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -372,7 +372,7 @@ F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c 00bbf37d80fb763a8159b83cc3cbde0f91d37f7b +F src/mutex_w32.c 3631e57d056062807ae2c92b79f3d1c05f04ecfe F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 @@ -1555,7 +1555,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e464b919f76520b45bb58983c6702db59d820ee4 -R 8c09ada0ea1970f0e52c7f9e3e6704e9 +P c447441cff1884d6fe5f0a76d64b3e7d908584a1 +Q +25ebadd096ce98fd0f9fab809969d34b958794f6 +R 0d77a6b89e0c4cc810773e50b9f58357 U drh -Z 6a635e248bfa56654adaa33accae463e +Z 66e01577a77088f99ecc22522d190a03 diff --git a/manifest.uuid b/manifest.uuid index a7d1b63450..e47a2c37d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c447441cff1884d6fe5f0a76d64b3e7d908584a1 \ No newline at end of file +f3b65926b1f439adb95e3bbce8e58785b8cf8427 \ No newline at end of file diff --git a/src/btreeInt.h b/src/btreeInt.h index 0d79edd639..b01163c33f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -694,7 +694,7 @@ struct IntegrityCk { # define get2byteAligned(x) (*(u16*)(x)) #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && MSCV_VERSION>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) diff --git a/src/mutex_w32.c b/src/mutex_w32.c index e0bdf9e43b..505c8fab7e 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -87,7 +87,7 @@ void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif MSCV_VERSION>=1300 +#elif MSVC_VERSION>=1300 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); From d879e3eb8d1326182ae33fc0ecd47563add7e2b5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2017 13:35:55 +0000 Subject: [PATCH 150/292] Change all legacy instances of "#if SQLITE_DEBUG" to "#ifdef SQLITE_DEBUG" for consistency. FossilOrigin-Name: 670f10b24230863688270d12ac519609ade2302b --- manifest | 25 ++++++++++++------------- manifest.uuid | 2 +- src/btree.c | 4 ++-- src/ctime.c | 2 +- src/expr.c | 2 +- src/malloc.c | 4 ++-- src/pcache.c | 2 +- src/vdbe.c | 2 +- src/vdbeaux.c | 4 ++-- 9 files changed, 23 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index fbd6c3cbf1..2ceff8b1f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\susing\sthe\sMSVC_VERSION\smacro. -D 2017-02-13T13:26:33.213 +C Change\sall\slegacy\sinstances\sof\s"#if\sSQLITE_DEBUG"\sto\s"#ifdef\sSQLITE_DEBUG"\sfor\nconsistency. +D 2017-02-13T13:35:55.526 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -337,17 +337,17 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 9fe65ab418d99e80289f024016b5e5e74c3059dd +F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 9f2296a4e5d26ebf0e0d95a0af4628f1ea694e7a +F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c 4748f51a680ded80332aca51cc51548c60758fe9 +F src/expr.c 38bd92fcbd86b3904bfa29e477412e374b7df5a2 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -360,7 +360,7 @@ F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 -F src/malloc.c f345c60d093d6c6cf451b99a7344a123b1a70fd9 +F src/malloc.c d0a1474236486165bcb349af82e2a6560178bf7b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -384,7 +384,7 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c -F src/pcache.c 51070ec9b8251bbf9c6ea3d35fd96a458752929e +F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d @@ -461,11 +461,11 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c 3d2ce209a89b95cf35bffa16440f57368576e2ee F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c e7b1e860140f1d1803c6f4176b1e8f3801f3a290 +F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 -F src/vdbeaux.c b9a36e530e6525ca9d9a685bc7b1d01fa77b5cf8 +F src/vdbeaux.c 4122458d33318ab039c4b5da1ca4e7c9221c38e4 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1555,8 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c447441cff1884d6fe5f0a76d64b3e7d908584a1 -Q +25ebadd096ce98fd0f9fab809969d34b958794f6 -R 0d77a6b89e0c4cc810773e50b9f58357 +P f3b65926b1f439adb95e3bbce8e58785b8cf8427 +R 9a737f02698a8160ed54e0df3f0a03b5 U drh -Z 66e01577a77088f99ecc22522d190a03 +Z 858958ba65336814a070740bd151ce33 diff --git a/manifest.uuid b/manifest.uuid index e47a2c37d8..897adb27e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3b65926b1f439adb95e3bbce8e58785b8cf8427 \ No newline at end of file +670f10b24230863688270d12ac519609ade2302b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index de553423b8..e78ffef1be 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4776,7 +4776,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ pCur, pCur->curPagerFlags); } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -6163,7 +6163,7 @@ static int fillInCell( ** Use a call to btreeParseCellPtr() to verify that the values above ** were computed correctly. */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG { CellInfo info; pPage->xParseCell(pPage, pCell, &info); diff --git a/src/ctime.c b/src/ctime.c index 969bbc73d2..3db3192095 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,7 +57,7 @@ static const char * const azCompileOpt[] = { #if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG "DEBUG", #endif #if SQLITE_DEFAULT_LOCKING_MODE diff --git a/src/expr.c b/src/expr.c index 6a1a9288a1..505e56a272 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3272,7 +3272,7 @@ void sqlite3ExprCodeGetColumnToReg( void sqlite3ExprCacheClear(Parse *pParse){ int i; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } diff --git a/src/malloc.c b/src/malloc.c index e7714aa103..0fdc8d73ff 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -418,7 +418,7 @@ int sqlite3MallocSize(void *p){ int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); if( db==0 || !isLookaside(db,p) ){ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -479,7 +479,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ } if( isLookaside(db, p) ){ LookasideSlot *pBuf = (LookasideSlot*)p; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* Trash all content in the buffer being freed */ memset(p, 0xaa, db->lookaside.sz); #endif diff --git a/src/pcache.c b/src/pcache.c index 0fc44c5499..dc7d00f306 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -104,7 +104,7 @@ struct PCache { ** ** assert( sqlite3PcachePageSanity(pPg) ); */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG int sqlite3PcachePageSanity(PgHdr *pPg){ PCache *pCache; assert( pPg!=0 ); diff --git a/src/vdbe.c b/src/vdbe.c index cb8a039abb..7f286b2c6d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2198,7 +2198,7 @@ case OP_Compare: { assert( pKeyInfo!=0 ); p1 = pOp->p1; p2 = pOp->p2; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( aPermute ){ int k, mx = 0; for(k=0; kmx ) mx = aPermute[k]; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 68f6a5acc8..cf062cf01d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3562,7 +3562,7 @@ void sqlite3VdbeRecordUnpack( p->nField = u; } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** This function compares two index or table record keys in the same way ** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), @@ -3667,7 +3667,7 @@ debugCompareEnd: } #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Count the number of fields (a.k.a. columns) in the record given by ** pKey,nKey. The verify that this count is less than or equal to the From a8207f488fdf5008516996b5b784f86f572f194a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Feb 2017 15:57:11 +0000 Subject: [PATCH 151/292] Add the new "--testset orm" to the speedtest1 utility. FossilOrigin-Name: 1836adc1d1f8e496ae0a07bf0fc933a19dc8fee5 --- manifest | 12 +- manifest.uuid | 2 +- test/speedtest1.c | 273 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 279 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2ceff8b1f9..209981b594 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sall\slegacy\sinstances\sof\s"#if\sSQLITE_DEBUG"\sto\s"#ifdef\sSQLITE_DEBUG"\sfor\nconsistency. -D 2017-02-13T13:35:55.526 +C Add\sthe\snew\s"--testset\sorm"\sto\sthe\sspeedtest1\sutility. +D 2017-02-14T15:57:11.586 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -1137,7 +1137,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 02fe15bb784c5276a083ffe9969cc51e0bce7644 +F test/speedtest1.c 89795b8e11f2e4d1e14882d5f03731f4bc49962a F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3b65926b1f439adb95e3bbce8e58785b8cf8427 -R 9a737f02698a8160ed54e0df3f0a03b5 +P 670f10b24230863688270d12ac519609ade2302b +R 2249f4f56d1658c4b20a6cc317c60c9d U drh -Z 858958ba65336814a070740bd151ce33 +Z 47dad88b095b1c4793f875e17e9e4c4b diff --git a/manifest.uuid b/manifest.uuid index 897adb27e8..eed33597ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -670f10b24230863688270d12ac519609ade2302b \ No newline at end of file +1836adc1d1f8e496ae0a07bf0fc933a19dc8fee5 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index f5b79915c8..075b2e24aa 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -33,7 +33,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --stats Show statistics at the end\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" - " --testset T Run test-set T\n" + " --testset T Run test-set T (main, cte, rtree, orm, debug)\n" " --trace Turn on SQL tracing\n" " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" @@ -1307,6 +1307,275 @@ void testset_rtree(int p1, int p2){ } #endif /* SQLITE_ENABLE_RTREE */ +/* +** A testset that does key/value storage on tables with many columns. +** This is the kind of workload generated by ORMs such as CoreData. +*/ +void testset_orm(void){ + unsigned i, j, n; + unsigned nRow; + unsigned x1, len; + char zNum[2000]; /* A number name */ + static const char zType[] = /* Types for all non-PK columns, in order */ + "IBBIIITIVVITBTBFBFITTFBTBVBVIFTBBFITFFVBIFIVBVVVBTVTIBBFFIVIBTB" + "TVTTFTVTVFFIITIFBITFTTFFFVBIIBTTITFTFFVVVFIIITVBBVFFTVVB"; + + nRow = n = g.szTest*250; + speedtest1_begin_test(100, "Fill %d rows", n); + speedtest1_exec( + "BEGIN;" + "CREATE TABLE ZLOOKSLIKECOREDATA (" + " ZPK INTEGER PRIMARY KEY," + " ZTERMFITTINGHOUSINGCOMMAND INTEGER," + " ZBRIEFGOBYDODGERHEIGHT BLOB," + " ZCAPABLETRIPDOORALMOND BLOB," + " ZDEPOSITPAIRCOLLEGECOMET INTEGER," + " ZFRAMEENTERSIMPLEMOUTH INTEGER," + " ZHOPEFULGATEHOLECHALK INTEGER," + " ZSLEEPYUSERGRANDBOWL TIMESTAMP," + " ZDEWPEACHCAREERCELERY INTEGER," + " ZHANGERLITHIUMDINNERMEET VARCHAR," + " ZCLUBRELEASELIZARDADVICE VARCHAR," + " ZCHARGECLICKHUMANEHIRE INTEGER," + " ZFINGERDUEPIZZAOPTION TIMESTAMP," + " ZFLYINGDOCTORTABLEMELODY BLOB," + " ZLONGFINLEAVEIMAGEOIL TIMESTAMP," + " ZFAMILYVISUALOWNERMATTER BLOB," + " ZGOLDYOUNGINITIALNOSE FLOAT," + " ZCAUSESALAMITERMCYAN BLOB," + " ZSPREADMOTORBISCUITBACON FLOAT," + " ZGIFTICEFISHGLUEHAIR INTEGER," + " ZNOTICEPEARPOLICYJUICE TIMESTAMP," + " ZBANKBUFFALORECOVERORBIT TIMESTAMP," + " ZLONGDIETESSAYNATURE FLOAT," + " ZACTIONRANGEELEGANTNEUTRON BLOB," + " ZCADETBRIGHTPLANETBANK TIMESTAMP," + " ZAIRFORGIVEHEADFROG BLOB," + " ZSHARKJUSTFRUITMOVIE VARCHAR," + " ZFARMERMORNINGMIRRORCONCERN BLOB," + " ZWOODPOETRYCOBBLERBENCH VARCHAR," + " ZHAFNIUMSCRIPTSALADMOTOR INTEGER," + " ZPROBLEMCLUBPOPOVERJELLY FLOAT," + " ZEIGHTLEADERWORKERMOST TIMESTAMP," + " ZGLASSRESERVEBARIUMMEAL BLOB," + " ZCLAMBITARUGULAFAJITA BLOB," + " ZDECADEJOYOUSWAVEHABIT FLOAT," + " ZCOMPANYSUMMERFIBERELF INTEGER," + " ZTREATTESTQUILLCHARGE TIMESTAMP," + " ZBROWBALANCEKEYCHOWDER FLOAT," + " ZPEACHCOPPERDINNERLAKE FLOAT," + " ZDRYWALLBEYONDBROWNBOWL VARCHAR," + " ZBELLYCRASHITEMLACK BLOB," + " ZTENNISCYCLEBILLOFFICER INTEGER," + " ZMALLEQUIPTHANKSGLUE FLOAT," + " ZMISSREPLYHUMANLIVING INTEGER," + " ZKIWIVISUALPRIDEAPPLE VARCHAR," + " ZWISHHITSKINMOTOR BLOB," + " ZCALMRACCOONPROGRAMDEBIT VARCHAR," + " ZSHINYASSISTLIVINGCRAB VARCHAR," + " ZRESOLVEWRISTWRAPAPPLE VARCHAR," + " ZAPPEALSIMPLESECONDHOUSING BLOB," + " ZCORNERANCHORTAPEDIVER TIMESTAMP," + " ZMEMORYREQUESTSOURCEBIG VARCHAR," + " ZTRYFACTKEEPMILK TIMESTAMP," + " ZDIVERPAINTLEATHEREASY INTEGER," + " ZSORTMISTYQUOTECABBAGE BLOB," + " ZTUNEGASBUFFALOCAPITAL BLOB," + " ZFILLSTOPLAWJOYFUL FLOAT," + " ZSTEELCAREFULPLATENUMBER FLOAT," + " ZGIVEVIVIDDIVINEMEANING INTEGER," + " ZTREATPACKFUTURECONVERT VARCHAR," + " ZCALMLYGEMFINISHEFFECT INTEGER," + " ZCABBAGESOCKEASEMINUTE BLOB," + " ZPLANETFAMILYPUREMEMORY TIMESTAMP," + " ZMERRYCRACKTRAINLEADER BLOB," + " ZMINORWAYPAPERCLASSY TIMESTAMP," + " ZEAGLELINEMINEMAIL VARCHAR," + " ZRESORTYARDGREENLET TIMESTAMP," + " ZYARDOREGANOVIVIDJEWEL TIMESTAMP," + " ZPURECAKEVIVIDNEATLY FLOAT," + " ZASKCONTACTMONITORFUN TIMESTAMP," + " ZMOVEWHOGAMMAINCH VARCHAR," + " ZLETTUCEBIRDMEETDEBATE TIMESTAMP," + " ZGENENATURALHEARINGKITE VARCHAR," + " ZMUFFINDRYERDRAWFORTUNE FLOAT," + " ZGRAYSURVEYWIRELOVE FLOAT," + " ZPLIERSPRINTASKOREGANO INTEGER," + " ZTRAVELDRIVERCONTESTLILY INTEGER," + " ZHUMORSPICESANDKIDNEY TIMESTAMP," + " ZARSENICSAMPLEWAITMUON INTEGER," + " ZLACEADDRESSGROUNDCAREFUL FLOAT," + " ZBAMBOOMESSWASABIEVENING BLOB," + " ZONERELEASEAVERAGENURSE INTEGER," + " ZRADIANTWHENTRYCARD TIMESTAMP," + " ZREWARDINSIDEMANGOINTENSE FLOAT," + " ZNEATSTEWPARTIRON TIMESTAMP," + " ZOUTSIDEPEAHENCOUNTICE TIMESTAMP," + " ZCREAMEVENINGLIPBRANCH FLOAT," + " ZWHALEMATHAVOCADOCOPPER FLOAT," + " ZLIFEUSELEAFYBELL FLOAT," + " ZWEALTHLINENGLEEFULDAY VARCHAR," + " ZFACEINVITETALKGOLD BLOB," + " ZWESTAMOUNTAFFECTHEARING INTEGER," + " ZDELAYOUTCOMEHORNAGENCY INTEGER," + " ZBIGTHINKCONVERTECONOMY BLOB," + " ZBASEGOUDAREGULARFORGIVE TIMESTAMP," + " ZPATTERNCLORINEGRANDCOLBY TIMESTAMP," + " ZCYANBASEFEEDADROIT INTEGER," + " ZCARRYFLOORMINNOWDRAGON TIMESTAMP," + " ZIMAGEPENCILOTHERBOTTOM FLOAT," + " ZXENONFLIGHTPALEAPPLE TIMESTAMP," + " ZHERRINGJOKEFEATUREHOPEFUL FLOAT," + " ZCAPYEARLYRIVETBRUSH FLOAT," + " ZAGEREEDFROGBASKET VARCHAR," + " ZUSUALBODYHALIBUTDIAMOND VARCHAR," + " ZFOOTTAPWORDENTRY VARCHAR," + " ZDISHKEEPBLESTMONITOR FLOAT," + " ZBROADABLESOLIDCASUAL INTEGER," + " ZSQUAREGLEEFULCHILDLIGHT INTEGER," + " ZHOLIDAYHEADPONYDETAIL INTEGER," + " ZGENERALRESORTSKYOPEN TIMESTAMP," + " ZGLADSPRAYKIDNEYGUPPY VARCHAR," + " ZSWIMHEAVYMENTIONKIND BLOB," + " ZMESSYSULFURDREAMFESTIVE BLOB," + " ZSKYSKYCLASSICBRIEF VARCHAR," + " ZDILLASKHOKILEMON FLOAT," + " ZJUNIORSHOWPRESSNOVA FLOAT," + " ZSIZETOEAWARDFRESH TIMESTAMP," + " ZKEYFAILAPRICOTMETAL VARCHAR," + " ZHANDYREPAIRPROTONAIRPORT VARCHAR," + " ZPOSTPROTEINHANDLEACTOR BLOB" + ");" + ); + speedtest1_prepare( + "INSERT INTO ZLOOKSLIKECOREDATA(ZPK,ZAIRFORGIVEHEADFROG," + "ZGIFTICEFISHGLUEHAIR,ZDELAYOUTCOMEHORNAGENCY,ZSLEEPYUSERGRANDBOWL," + "ZGLASSRESERVEBARIUMMEAL,ZBRIEFGOBYDODGERHEIGHT," + "ZBAMBOOMESSWASABIEVENING,ZFARMERMORNINGMIRRORCONCERN," + "ZTREATPACKFUTURECONVERT,ZCAUSESALAMITERMCYAN,ZCALMRACCOONPROGRAMDEBIT," + "ZHOLIDAYHEADPONYDETAIL,ZWOODPOETRYCOBBLERBENCH,ZHAFNIUMSCRIPTSALADMOTOR," + "ZUSUALBODYHALIBUTDIAMOND,ZOUTSIDEPEAHENCOUNTICE,ZDIVERPAINTLEATHEREASY," + "ZWESTAMOUNTAFFECTHEARING,ZSIZETOEAWARDFRESH,ZDEWPEACHCAREERCELERY," + "ZSTEELCAREFULPLATENUMBER,ZCYANBASEFEEDADROIT,ZCALMLYGEMFINISHEFFECT," + "ZHANDYREPAIRPROTONAIRPORT,ZGENENATURALHEARINGKITE,ZBROADABLESOLIDCASUAL," + "ZPOSTPROTEINHANDLEACTOR,ZLACEADDRESSGROUNDCAREFUL,ZIMAGEPENCILOTHERBOTTOM," + "ZPROBLEMCLUBPOPOVERJELLY,ZPATTERNCLORINEGRANDCOLBY,ZNEATSTEWPARTIRON," + "ZAPPEALSIMPLESECONDHOUSING,ZMOVEWHOGAMMAINCH,ZTENNISCYCLEBILLOFFICER," + "ZSHARKJUSTFRUITMOVIE,ZKEYFAILAPRICOTMETAL,ZCOMPANYSUMMERFIBERELF," + "ZTERMFITTINGHOUSINGCOMMAND,ZRESORTYARDGREENLET,ZCABBAGESOCKEASEMINUTE," + "ZSQUAREGLEEFULCHILDLIGHT,ZONERELEASEAVERAGENURSE,ZBIGTHINKCONVERTECONOMY," + "ZPLIERSPRINTASKOREGANO,ZDECADEJOYOUSWAVEHABIT,ZDRYWALLBEYONDBROWNBOWL," + "ZCLUBRELEASELIZARDADVICE,ZWHALEMATHAVOCADOCOPPER,ZBELLYCRASHITEMLACK," + "ZLETTUCEBIRDMEETDEBATE,ZCAPABLETRIPDOORALMOND,ZRADIANTWHENTRYCARD," + "ZCAPYEARLYRIVETBRUSH,ZAGEREEDFROGBASKET,ZSWIMHEAVYMENTIONKIND," + "ZTRAVELDRIVERCONTESTLILY,ZGLADSPRAYKIDNEYGUPPY,ZBANKBUFFALORECOVERORBIT," + "ZFINGERDUEPIZZAOPTION,ZCLAMBITARUGULAFAJITA,ZLONGFINLEAVEIMAGEOIL," + "ZLONGDIETESSAYNATURE,ZJUNIORSHOWPRESSNOVA,ZHOPEFULGATEHOLECHALK," + "ZDEPOSITPAIRCOLLEGECOMET,ZWEALTHLINENGLEEFULDAY,ZFILLSTOPLAWJOYFUL," + "ZTUNEGASBUFFALOCAPITAL,ZGRAYSURVEYWIRELOVE,ZCORNERANCHORTAPEDIVER," + "ZREWARDINSIDEMANGOINTENSE,ZCADETBRIGHTPLANETBANK,ZPLANETFAMILYPUREMEMORY," + "ZTREATTESTQUILLCHARGE,ZCREAMEVENINGLIPBRANCH,ZSKYSKYCLASSICBRIEF," + "ZARSENICSAMPLEWAITMUON,ZBROWBALANCEKEYCHOWDER,ZFLYINGDOCTORTABLEMELODY," + "ZHANGERLITHIUMDINNERMEET,ZNOTICEPEARPOLICYJUICE,ZSHINYASSISTLIVINGCRAB," + "ZLIFEUSELEAFYBELL,ZFACEINVITETALKGOLD,ZGENERALRESORTSKYOPEN," + "ZPURECAKEVIVIDNEATLY,ZKIWIVISUALPRIDEAPPLE,ZMESSYSULFURDREAMFESTIVE," + "ZCHARGECLICKHUMANEHIRE,ZHERRINGJOKEFEATUREHOPEFUL,ZYARDOREGANOVIVIDJEWEL," + "ZFOOTTAPWORDENTRY,ZWISHHITSKINMOTOR,ZBASEGOUDAREGULARFORGIVE," + "ZMUFFINDRYERDRAWFORTUNE,ZACTIONRANGEELEGANTNEUTRON,ZTRYFACTKEEPMILK," + "ZPEACHCOPPERDINNERLAKE,ZFRAMEENTERSIMPLEMOUTH,ZMERRYCRACKTRAINLEADER," + "ZMEMORYREQUESTSOURCEBIG,ZCARRYFLOORMINNOWDRAGON,ZMINORWAYPAPERCLASSY," + "ZDILLASKHOKILEMON,ZRESOLVEWRISTWRAPAPPLE,ZASKCONTACTMONITORFUN," + "ZGIVEVIVIDDIVINEMEANING,ZEIGHTLEADERWORKERMOST,ZMISSREPLYHUMANLIVING," + "ZXENONFLIGHTPALEAPPLE,ZSORTMISTYQUOTECABBAGE,ZEAGLELINEMINEMAIL," + "ZFAMILYVISUALOWNERMATTER,ZSPREADMOTORBISCUITBACON,ZDISHKEEPBLESTMONITOR," + "ZMALLEQUIPTHANKSGLUE,ZGOLDYOUNGINITIALNOSE,ZHUMORSPICESANDKIDNEY)" + "VALUES(?1,?26,?20,?93,?8,?33,?3,?81,?28,?60,?18,?47,?109,?29,?30,?104,?86," + "?54,?92,?117,?9,?58,?97,?61,?119,?73,?107,?120,?80,?99,?31,?96,?85,?50,?71," + "?42,?27,?118,?36,?2,?67,?62,?108,?82,?94,?76,?35,?40,?11,?88,?41,?72,?4," + "?83,?102,?103,?112,?77,?111,?22,?13,?34,?15,?23,?116,?7,?5,?90,?57,?56," + "?75,?51,?84,?25,?63,?37,?87,?114,?79,?38,?14,?10,?21,?48,?89,?91,?110," + "?69,?45,?113,?12,?101,?68,?105,?46,?95,?74,?24,?53,?39,?6,?64,?52,?98," + "?65,?115,?49,?70,?59,?32,?44,?100,?55,?66,?16,?19,?106,?43,?17,?78);" + ); + for(i=0; i Date: Tue, 14 Feb 2017 15:58:58 +0000 Subject: [PATCH 152/292] Fix a testcase number on the ORM testset of speedtest1. FossilOrigin-Name: 58b2f911eec2e3eb9944dd6d8573ff5c7bd43f70 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 209981b594..eae1f1a83e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\snew\s"--testset\sorm"\sto\sthe\sspeedtest1\sutility. -D 2017-02-14T15:57:11.586 +C Fix\sa\stestcase\snumber\son\sthe\sORM\stestset\sof\sspeedtest1. +D 2017-02-14T15:58:58.551 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -1137,7 +1137,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 89795b8e11f2e4d1e14882d5f03731f4bc49962a +F test/speedtest1.c 4f7a10a3db37a5d9ab673244b8c87f77fa8ab913 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 670f10b24230863688270d12ac519609ade2302b -R 2249f4f56d1658c4b20a6cc317c60c9d +P 1836adc1d1f8e496ae0a07bf0fc933a19dc8fee5 +R db256bb9d4f2af476591a168e0c752ad U drh -Z 47dad88b095b1c4793f875e17e9e4c4b +Z 52eb4a5120830a27e32049ce6ccc32b2 diff --git a/manifest.uuid b/manifest.uuid index eed33597ba..a19db54686 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1836adc1d1f8e496ae0a07bf0fc933a19dc8fee5 \ No newline at end of file +58b2f911eec2e3eb9944dd6d8573ff5c7bd43f70 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 075b2e24aa..2117e66ce3 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -1524,7 +1524,7 @@ void testset_orm(void){ speedtest1_end_test(); n = g.szTest*250; - speedtest1_begin_test(100, "Query %d rows by rowid", n); + speedtest1_begin_test(110, "Query %d rows by rowid", n); speedtest1_prepare( "SELECT ZCYANBASEFEEDADROIT,ZJUNIORSHOWPRESSNOVA,ZCAUSESALAMITERMCYAN," "ZHOPEFULGATEHOLECHALK,ZHUMORSPICESANDKIDNEY,ZSWIMHEAVYMENTIONKIND," From d2f92c26d58a12761a3a2d6cfdf17ad50a007932 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Feb 2017 16:30:13 +0000 Subject: [PATCH 153/292] More realistic lengths of string values in speedtest1 with --testset orm. FossilOrigin-Name: e4731fd65f9698817690b741cc454f25e8e871e6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index eae1f1a83e..9b10871ba3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stestcase\snumber\son\sthe\sORM\stestset\sof\sspeedtest1. -D 2017-02-14T15:58:58.551 +C More\srealistic\slengths\sof\sstring\svalues\sin\sspeedtest1\swith\s--testset\sorm. +D 2017-02-14T16:30:13.697 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -1137,7 +1137,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 4f7a10a3db37a5d9ab673244b8c87f77fa8ab913 +F test/speedtest1.c 7b1ab42b097b484c18d99e1d1c71a6a0c9c87a7a F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1836adc1d1f8e496ae0a07bf0fc933a19dc8fee5 -R db256bb9d4f2af476591a168e0c752ad +P 58b2f911eec2e3eb9944dd6d8573ff5c7bd43f70 +R 20b26eda0719e5594bae6bc77f8993f3 U drh -Z 52eb4a5120830a27e32049ce6ccc32b2 +Z ceef40c49fe3ba25325b7e3939c1bded diff --git a/manifest.uuid b/manifest.uuid index a19db54686..38bf6ebaca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58b2f911eec2e3eb9944dd6d8573ff5c7bd43f70 \ No newline at end of file +e4731fd65f9698817690b741cc454f25e8e871e6 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 2117e66ce3..db3a558a38 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -1499,14 +1499,14 @@ void testset_orm(void){ ); for(i=0; i Date: Tue, 14 Feb 2017 20:00:16 +0000 Subject: [PATCH 154/292] Enable the SQLITE_ENABLE_NULL_TRIM option for WITHOUT ROWID tables. FossilOrigin-Name: 54836270c9c0bfa5910f7ad74ec238b9d7ddee5f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/insert.c | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9b10871ba3..66d9fc8753 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\srealistic\slengths\sof\sstring\svalues\sin\sspeedtest1\swith\s--testset\sorm. -D 2017-02-14T16:30:13.697 +C Enable\sthe\sSQLITE_ENABLE_NULL_TRIM\soption\sfor\sWITHOUT\sROWID\stables. +D 2017-02-14T20:00:16.259 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -356,7 +356,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 444354c23d4d140a57d6eb46f34e376a7f8f62e8 +F src/insert.c 891f6789bafca1f0a3b4f7cbb9c363229eaec828 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58b2f911eec2e3eb9944dd6d8573ff5c7bd43f70 -R 20b26eda0719e5594bae6bc77f8993f3 +P e4731fd65f9698817690b741cc454f25e8e871e6 +R 239d7ae4b4c76fa68b6a3d5959c5ddbe U drh -Z ceef40c49fe3ba25325b7e3939c1bded +Z 558f1297c34d38d9f2c9884a0e7f3db0 diff --git a/manifest.uuid b/manifest.uuid index 38bf6ebaca..545a5042ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4731fd65f9698817690b741cc454f25e8e871e6 \ No newline at end of file +54836270c9c0bfa5910f7ad74ec238b9d7ddee5f \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 894bfc2cc1..897a048ac9 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1527,6 +1527,9 @@ void sqlite3GenerateConstraintChecks( } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==2 ) sqlite3SetMakeRecordP5(v, pIdx->pTable); +#endif /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the @@ -1682,8 +1685,11 @@ void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ if( pTab->pSchema->file_format<2 ) return; - for(i=pTab->nCol; i>1 && pTab->aCol[i-1].pDflt==0; i--){} - sqlite3VdbeChangeP5(v, i); + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].pDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); } #endif From 0c5cd969b8930871b2405790c6be5cb2220a2462 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Feb 2017 21:47:46 +0000 Subject: [PATCH 155/292] Clarification of the help text for the command-line shell. FossilOrigin-Name: ca4f1e4962df64ae756c286f3795af7d6f692cdd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 66d9fc8753..28095b65b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\sSQLITE_ENABLE_NULL_TRIM\soption\sfor\sWITHOUT\sROWID\stables. -D 2017-02-14T20:00:16.259 +C Clarification\sof\sthe\shelp\stext\sfor\sthe\scommand-line\sshell. +D 2017-02-14T21:47:46.389 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -395,7 +395,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c a84e453c213f3e0d6935a582024da4e242f85a19 +F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e4731fd65f9698817690b741cc454f25e8e871e6 -R 239d7ae4b4c76fa68b6a3d5959c5ddbe +P 54836270c9c0bfa5910f7ad74ec238b9d7ddee5f +R 81ac83b9aa3d80dbb34372d0527e01bd U drh -Z 558f1297c34d38d9f2c9884a0e7f3db0 +Z 42f594c3c0159a4f4af8ef6e693c76a8 diff --git a/manifest.uuid b/manifest.uuid index 545a5042ea..da9b31e873 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54836270c9c0bfa5910f7ad74ec238b9d7ddee5f \ No newline at end of file +ca4f1e4962df64ae756c286f3795af7d6f692cdd \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 0e553d9fbb..fe62bfc8a1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2212,7 +2212,7 @@ static char zHelp[] = " html HTML code\n" " insert SQL insert statements for TABLE\n" " line One value per line\n" - " list Values delimited by .separator strings\n" + " list Values delimited by \"|\"\n" " quote Escape answers as for SQL\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" From dd22c09af8ab8208a676effac377df1969cdb1a3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 15 Feb 2017 01:39:28 +0000 Subject: [PATCH 156/292] In the blob test code, avoid crashing on low-memory systems by using Tcl_AttemptAlloc(). FossilOrigin-Name: 1d267757a89d9267ee9c201373f801eb9772ab04 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_blob.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 28095b65b6..1dd8b1f184 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarification\sof\sthe\shelp\stext\sfor\sthe\scommand-line\sshell. -D 2017-02-14T21:47:46.389 +C In\sthe\sblob\stest\scode,\savoid\scrashing\son\slow-memory\ssystems\sby\susing\sTcl_AttemptAlloc(). +D 2017-02-15T01:39:28.000 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -417,7 +417,7 @@ F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 -F src/test_blob.c 6a4c7920d1d9c6cc0f7aa50c89c4f80016aeda83 +F src/test_blob.c f65ac717da2618691cf9dad094e6da0219dcd208 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 83179ea845479b5be9a651d014649e3f2722a1fe F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54836270c9c0bfa5910f7ad74ec238b9d7ddee5f -R 81ac83b9aa3d80dbb34372d0527e01bd -U drh -Z 42f594c3c0159a4f4af8ef6e693c76a8 +P ca4f1e4962df64ae756c286f3795af7d6f692cdd +R 91e8830a4e86062346b40a8d7a630a58 +U mistachkin +Z 631cc99fc676ac52e315c4b1825161e1 diff --git a/manifest.uuid b/manifest.uuid index da9b31e873..50375fa2e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca4f1e4962df64ae756c286f3795af7d6f692cdd \ No newline at end of file +1d267757a89d9267ee9c201373f801eb9772ab04 \ No newline at end of file diff --git a/src/test_blob.c b/src/test_blob.c index 7fa733bee9..118f210738 100644 --- a/src/test_blob.c +++ b/src/test_blob.c @@ -239,7 +239,11 @@ static int SQLITE_TCLAPI test_blob_read( } if( nByte>0 ){ - zBuf = (unsigned char *)Tcl_Alloc(nByte); + zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte); + if( zBuf==0 ){ + Tcl_AppendResult(interp, "out of memory", 0); + return TCL_ERROR; + } } rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); if( rc==SQLITE_OK ){ From d742367ab58014c121fb450797c4832d795af3eb Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 15 Feb 2017 04:16:56 +0000 Subject: [PATCH 157/292] Further reforms to Tcl_*Alloc() usage. FossilOrigin-Name: ee1e689633e517ce46307b9afbf1eda03482c928 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/tclsqlite.c | 4 ---- src/test6.c | 4 ++-- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 1dd8b1f184..755e760073 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sblob\stest\scode,\savoid\scrashing\son\slow-memory\ssystems\sby\susing\sTcl_AttemptAlloc(). -D 2017-02-15T01:39:28.000 +C Further\sreforms\sto\sTcl_*Alloc()\susage. +D 2017-02-15T04:16:56.208 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -403,13 +403,13 @@ F src/sqliteInt.h c3f878dcbe947938f9e0984644f1902dd9051094 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 418f5e5e0840425a7e5b33f3600dccd378a57549 +F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d -F src/test6.c 55aa2775c154415dcf4ed7cd1e19a193122b3a02 +F src/test6.c 121060d2e79a4f5047eb12b5135b23a6a7a5af01 F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010 F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86 F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca4f1e4962df64ae756c286f3795af7d6f692cdd -R 91e8830a4e86062346b40a8d7a630a58 +P 1d267757a89d9267ee9c201373f801eb9772ab04 +R f321de2fcbd363565a0432a788265485 U mistachkin -Z 631cc99fc676ac52e315c4b1825161e1 +Z c20a15243be58007e25bc4aac5180d1c diff --git a/manifest.uuid b/manifest.uuid index 50375fa2e4..5be845459d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d267757a89d9267ee9c201373f801eb9772ab04 \ No newline at end of file +ee1e689633e517ce46307b9afbf1eda03482c928 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 5b52bf0c91..9df023b45c 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3405,10 +3405,6 @@ static int SQLITE_TCLAPI DbMain( } zErrMsg = 0; p = (SqliteDb*)Tcl_Alloc( sizeof(*p) ); - if( p==0 ){ - Tcl_SetResult(interp, (char *)"malloc failed", TCL_STATIC); - return TCL_ERROR; - } memset(p, 0, sizeof(*p)); zFile = Tcl_GetStringFromObj(objv[2], 0); zFile = Tcl_TranslateFileName(interp, zFile, &translatedFilename); diff --git a/src/test6.c b/src/test6.c index 5304bcc31f..a103b9619e 100644 --- a/src/test6.c +++ b/src/test6.c @@ -161,13 +161,13 @@ static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0}; static int sqlite3CrashTestEnable = 0; static void *crash_malloc(int nByte){ - return (void *)Tcl_Alloc((size_t)nByte); + return (void *)Tcl_AttemptAlloc((size_t)nByte); } static void crash_free(void *p){ Tcl_Free(p); } static void *crash_realloc(void *p, int n){ - return (void *)Tcl_Realloc(p, (size_t)n); + return (void *)Tcl_AttemptRealloc(p, (size_t)n); } /* From dc5ece86ae370248de47a64ea1a022a2fa27e999 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Feb 2017 15:09:09 +0000 Subject: [PATCH 158/292] Remove the CLANG_VERSION macro, since we have learned that version numbers in clang are "marketing" and are inconsistent and unreliable. Builds using clang will still use the GCC_VERSION macro since clang works hard to be gcc compatible. FossilOrigin-Name: 810d29320b853b3a01aa50d8f2a0bceacf79e0aa --- ext/rtree/rtree.c | 26 ++++++++++---------------- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/sqliteInt.h | 15 ++++++++------- src/util.c | 10 +++++----- 5 files changed, 33 insertions(+), 38 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 205939ddc6..63838a4eef 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -368,7 +368,11 @@ struct RtreeMatchArg { # define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif -/* What version of GCC is being used. 0 means GCC is not being used */ +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ #ifndef GCC_VERSION #if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) @@ -377,16 +381,6 @@ struct RtreeMatchArg { #endif #endif -/* What version of CLANG is being used. 0 means CLANG is not being used */ -#ifndef CLANG_VERSION -#if defined(__clang__) && !defined(_WIN32) && !defined(SQLITE_DISABLE_INTRINSIC) -# define CLANG_VERSION \ - (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) -#else -# define CLANG_VERSION 0 -#endif -#endif - /* The testcase() macro should already be defined in the amalgamation. If ** it is not, make it a no-op. */ @@ -437,7 +431,7 @@ static void readCoord(u8 *p, RtreeCoord *pCoord){ assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 pCoord->u = _byteswap_ulong(*(u32*)p); -#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 pCoord->u = __builtin_bswap32(*(u32*)p); #elif SQLITE_BYTEORDER==4321 pCoord->u = *(u32*)p; @@ -455,7 +449,7 @@ static i64 readInt64(u8 *p){ u64 x; memcpy(&x, p, 8); return (i64)_byteswap_uint64(x); -#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u64 x; memcpy(&x, p, 8); return (i64)__builtin_bswap64(x); @@ -491,7 +485,7 @@ static int writeCoord(u8 *p, RtreeCoord *pCoord){ assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); -#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 i = __builtin_bswap32(pCoord->u); memcpy(p, &i, 4); #elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 @@ -510,7 +504,7 @@ static int writeCoord(u8 *p, RtreeCoord *pCoord){ return 4; } static int writeInt64(u8 *p, i64 i){ -#if SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 i = (i64)__builtin_bswap64((u64)i); memcpy(p, &i, 8); #elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 @@ -1066,7 +1060,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ c.u = _byteswap_ulong(*(u32*)a); \ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } -#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ c.u = __builtin_bswap32(*(u32*)a); \ diff --git a/manifest b/manifest index 755e760073..c24bc286f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sreforms\sto\sTcl_*Alloc()\susage. -D 2017-02-15T04:16:56.208 +C Remove\sthe\sCLANG_VERSION\smacro,\ssince\swe\shave\slearned\sthat\sversion\snumbers\sin\nclang\sare\s"marketing"\sand\sare\sinconsistent\sand\sunreliable.\s\sBuilds\susing\sclang\nwill\sstill\suse\sthe\sGCC_VERSION\smacro\ssince\sclang\sworks\shard\sto\sbe\sgcc\ncompatible. +D 2017-02-15T15:09:09.031 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -264,7 +264,7 @@ F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 358796a385de74e40ed66c103df77ba3c2e98fab +F ext/rtree/rtree.c 3f3a595dba485e340246fa2c8ba330a6b9768b00 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -399,7 +399,7 @@ F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h c3f878dcbe947938f9e0984644f1902dd9051094 +F src/sqliteInt.h 4f85005b109c1a7eab3110cb4568fe30a5389bda F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -459,7 +459,7 @@ F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 3d2ce209a89b95cf35bffa16440f57368576e2ee +F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d267757a89d9267ee9c201373f801eb9772ab04 -R f321de2fcbd363565a0432a788265485 -U mistachkin -Z c20a15243be58007e25bc4aac5180d1c +P ee1e689633e517ce46307b9afbf1eda03482c928 +R 09fd15812127544dc0546a143adbc35f +U drh +Z f21515d79bd695dc7bd9e01927424a91 diff --git a/manifest.uuid b/manifest.uuid index 5be845459d..f9059a7c49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee1e689633e517ce46307b9afbf1eda03482c928 \ No newline at end of file +810d29320b853b3a01aa50d8f2a0bceacf79e0aa \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 23b4dd9460..a65bb25a4d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -103,23 +103,24 @@ # define _LARGEFILE_SOURCE 1 #endif -/* The GCC_VERSION, CLANG_VERSION, and MSVC_VERSION macros are used to +/* The GCC_VERSION and MSVC_VERSION macros are used to ** conditionally include optimizations for each of these compilers. A ** value of 0 means that compiler is not being used. The ** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific ** optimizations, and hence set all compiler macros to 0 +** +** There was once also a CLANG_VERSION macro. However, we learn that the +** version numbers in clang are for "marketing" only and are inconsistent +** and unreliable. Fortunately, all versions of clang also recognize the +** gcc version numbers and have reasonable settings for gcc version numbers, +** so the GCC_VERSION macro will be set to a correct non-zero value even +** when compiling with clang. */ #if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) #else # define GCC_VERSION 0 #endif -#if defined(__clang__) && !defined(_WIN32) && !defined(SQLITE_DISABLE_INTRINSIC) -# define CLANG_VERSION \ - (__clang_major__*1000000+__clang_minor__*1000+__clang_patchlevel__) -#else -# define CLANG_VERSION 0 -#endif #if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) # define MSVC_VERSION _MSC_VER #else diff --git a/src/util.c b/src/util.c index c6d2bae3a7..9b6f4f9e1d 100644 --- a/src/util.c +++ b/src/util.c @@ -1140,7 +1140,7 @@ u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); @@ -1156,7 +1156,7 @@ u32 sqlite3Get4byte(const u8 *p){ void sqlite3Put4byte(unsigned char *p, u32 v){ #if SQLITE_BYTEORDER==4321 memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && (GCC_VERSION>=4003000 || CLANG_VERSION>=3000000) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x = __builtin_bswap32(v); memcpy(p,&x,4); #elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 @@ -1275,7 +1275,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ ** overflow, leave *pA unchanged and return 1. */ int sqlite3AddInt64(i64 *pA, i64 iB){ -#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 +#if GCC_VERSION>=5004000 return __builtin_add_overflow(*pA, iB, pA); #else i64 iA = *pA; @@ -1295,7 +1295,7 @@ int sqlite3AddInt64(i64 *pA, i64 iB){ #endif } int sqlite3SubInt64(i64 *pA, i64 iB){ -#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 +#if GCC_VERSION>=5004000 return __builtin_sub_overflow(*pA, iB, pA); #else testcase( iB==SMALLEST_INT64+1 ); @@ -1310,7 +1310,7 @@ int sqlite3SubInt64(i64 *pA, i64 iB){ #endif } int sqlite3MulInt64(i64 *pA, i64 iB){ -#if GCC_VERSION>=5004000 || CLANG_VERSION>=4000000 +#if GCC_VERSION>=5004000 return __builtin_mul_overflow(*pA, iB, pA); #else i64 iA = *pA; From 0d5b3b7665f895942be4002d46c2e0cf20765f8f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 15 Feb 2017 18:30:57 +0000 Subject: [PATCH 159/292] Minor enhancement to mutex tracing on Win32. FossilOrigin-Name: 830b9235673be55f0c932fb157de03725e648c25 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index c24bc286f6..7421867390 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sCLANG_VERSION\smacro,\ssince\swe\shave\slearned\sthat\sversion\snumbers\sin\nclang\sare\s"marketing"\sand\sare\sinconsistent\sand\sunreliable.\s\sBuilds\susing\sclang\nwill\sstill\suse\sthe\sGCC_VERSION\smacro\ssince\sclang\sworks\shard\sto\sbe\sgcc\ncompatible. -D 2017-02-15T15:09:09.031 +C Minor\senhancement\sto\smutex\stracing\son\sWin32. +D 2017-02-15T18:30:57.375 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -372,7 +372,7 @@ F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c 3631e57d056062807ae2c92b79f3d1c05f04ecfe +F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 @@ -1555,7 +1555,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee1e689633e517ce46307b9afbf1eda03482c928 -R 09fd15812127544dc0546a143adbc35f -U drh -Z f21515d79bd695dc7bd9e01927424a91 +P 810d29320b853b3a01aa50d8f2a0bceacf79e0aa +R 91fee55f54fe2e54254ba44f0240778d +U mistachkin +Z 72fbc76e1b27f40edc76050b0de83496 diff --git a/manifest.uuid b/manifest.uuid index f9059a7c49..54948777cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -810d29320b853b3a01aa50d8f2a0bceacf79e0aa \ No newline at end of file +830b9235673be55f0c932fb157de03725e648c25 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 505c8fab7e..9da93cf319 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -298,8 +298,8 @@ static void winMutexEnter(sqlite3_mutex *p){ p->owner = tid; p->nRef++; if( p->trace ){ - OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } @@ -341,8 +341,8 @@ static int winMutexTry(sqlite3_mutex *p){ #endif #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", - tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + OSTRACE(("TRY-MUTEX tid=%lu, mutex(%d)=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p->id, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -370,8 +370,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } From 3a3b420abba7630d47eb28636bd59ca1fb319819 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Feb 2017 22:36:15 +0000 Subject: [PATCH 160/292] Query planner optimization to detect empty tables in a join early and bail out without doing excess work. FossilOrigin-Name: 58797e9bafa95709e0f706a15f42f93b409e2db5 --- manifest | 15 ++++++------- manifest.uuid | 2 +- src/wherecode.c | 10 +++++++-- test/emptytable.test | 50 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 test/emptytable.test diff --git a/manifest b/manifest index 7421867390..b98e432695 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\senhancement\sto\smutex\stracing\son\sWin32. -D 2017-02-15T18:30:57.375 +C Query\splanner\soptimization\sto\sdetect\sempty\stables\sin\sa\sjoin\searly\sand\sbail\sout\nwithout\sdoing\sexcess\swork. +D 2017-02-15T22:36:15.061 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -477,7 +477,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 6397fab50fdbf9bde76c574ce07b3b776eb28b34 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d -F src/wherecode.c 99a8ced164c75edf41b3a865a75381c9adb38b28 +F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -676,6 +676,7 @@ F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 4c0613a0c76e7a9d5c4c211e1b4cbcc1143914df +F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6 @@ -1555,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 810d29320b853b3a01aa50d8f2a0bceacf79e0aa -R 91fee55f54fe2e54254ba44f0240778d -U mistachkin -Z 72fbc76e1b27f40edc76050b0de83496 +P 830b9235673be55f0c932fb157de03725e648c25 +R 32e948726be0c8ec843ec4b81e44685b +U drh +Z 345aeb62ab5f52d0700dc3107ec5c535 diff --git a/manifest.uuid b/manifest.uuid index 54948777cb..8b9eddf3bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -830b9235673be55f0c932fb157de03725e648c25 \ No newline at end of file +58797e9bafa95709e0f706a15f42f93b409e2db5 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index 58e040628a..4fd5e16fac 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1062,6 +1062,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( Vdbe *v; /* The prepared stmt under constructions */ struct SrcList_item *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -1103,6 +1104,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "init LEFT JOIN no-match flag")); } + /* Compute a safe address to jump to if we discover that the table for + ** this loop is empty and can never contribute content. */ + for(j=iLevel; j>0 && pWInfo->a[j].iLeftJoin==0; j--){} + addrHalt = pWInfo->a[j].addrBrk; + /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; @@ -1287,7 +1293,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -1933,7 +1939,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; diff --git a/test/emptytable.test b/test/emptytable.test new file mode 100644 index 0000000000..79cd16e913 --- /dev/null +++ b/test/emptytable.test @@ -0,0 +1,50 @@ +# 2017-02-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases to show that a join involving an empty table is very fast. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Build some test data +# +do_execsql_test emptytable-100 { + CREATE TABLE t1(a); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a) SELECT x FROM c; + CREATE TABLE empty(x); + SELECT count(*) FROM t1; +} {100} + +# Interrupt queries after 1M cycles to prevent burning excess CPU +proc stopDb {args} { + db interrupt +} +db progress 1000000 {stopDb} + +# Prior to the query planner optimization on 2017-02-15, this query would +# take a ridiculous amount of time. If that optimization stops working, +# the result here will be in interrupt for running too long. +# +do_catchsql_test emptytable-110 { + SELECT count(*) FROM t1, t1, t1, t1, t1, t1, empty; +} {0 0} + +do_catchsql_test emptytable-120 { + SELECT count(*) FROM t1, t1 LEFT JOIN empty; +} {0 10000} +do_catchsql_test emptytable-121 { + SELECT count(*) FROM t1, t1 LEFT JOIN t1, empty; +} {0 0} + + +finish_test From 44266ec651506ff7409a65852f21e5ebab5e5c94 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 14:48:08 +0000 Subject: [PATCH 161/292] Always use the IsVirtual() macro to determine if a Table object is a virtual table. Slightly smaller and faster code. FossilOrigin-Name: 6affb1c89d87288cad87dde5a533832cdf06b8aa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 4 ++-- src/sqliteInt.h | 4 ++-- src/vtab.c | 14 ++++++-------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index b98e432695..db4da4c16d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Query\splanner\soptimization\sto\sdetect\sempty\stables\sin\sa\sjoin\searly\sand\sbail\sout\nwithout\sdoing\sexcess\swork. -D 2017-02-15T22:36:15.061 +C Always\suse\sthe\sIsVirtual()\smacro\sto\sdetermine\sif\sa\sTable\sobject\sis\sa\svirtual\ntable.\s\sSlightly\ssmaller\sand\sfaster\scode. +D 2017-02-16T14:48:08.387 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -356,7 +356,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 891f6789bafca1f0a3b4f7cbb9c363229eaec828 +F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 @@ -399,7 +399,7 @@ F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 4f85005b109c1a7eab3110cb4568fe30a5389bda +F src/sqliteInt.h b4a3871bda47ff79b4be612a2ab89775fa737583 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -470,7 +470,7 @@ F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 -F src/vtab.c c4bbe0f870f52036553f8098aee0703997f0577a +F src/vtab.c f7f26f9db3c328824674d0ed4fd9a961a9919186 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 830b9235673be55f0c932fb157de03725e648c25 -R 32e948726be0c8ec843ec4b81e44685b +P 58797e9bafa95709e0f706a15f42f93b409e2db5 +R db3fb8708d0cf4bffde97f5d6d76493c U drh -Z 345aeb62ab5f52d0700dc3107ec5c535 +Z 8764b23fc3772844f5b9c915005803c6 diff --git a/manifest.uuid b/manifest.uuid index 8b9eddf3bb..c7ccabdfe1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58797e9bafa95709e0f706a15f42f93b409e2db5 \ No newline at end of file +6affb1c89d87288cad87dde5a533832cdf06b8aa \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 897a048ac9..f1f3807244 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1979,7 +1979,7 @@ static int xferOptimization( return 0; /* tab1 must not have triggers */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->tabFlags & TF_Virtual ){ + if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ } #endif @@ -2041,7 +2041,7 @@ static int xferOptimization( return 0; /* source and destination must both be WITHOUT ROWID or not */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->tabFlags & TF_Virtual ){ + if( IsVirtual(pSrc) ){ return 0; /* tab2 must not be a virtual table */ } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a65bb25a4d..c8f42e315c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1883,7 +1883,7 @@ struct Table { #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -#define TF_Virtual 0x10 /* Is a virtual table */ +/* available for reuse: 0x10 */ #define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ @@ -1895,7 +1895,7 @@ struct Table { ** table support is omitted from the build. */ #ifndef SQLITE_OMIT_VIRTUALTABLE -# define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) +# define IsVirtual(X) ((X)->nModuleArg) #else # define IsVirtual(X) 0 #endif diff --git a/src/vtab.c b/src/vtab.c index 30079cbb5e..d68ec8f627 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -339,7 +339,6 @@ void sqlite3VtabBeginParse( iDb = sqlite3SchemaToIndex(db, pTable->pSchema); assert( iDb>=0 ); - pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); @@ -628,7 +627,7 @@ int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ int rc; assert( pTab ); - if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ + if( !IsVirtual(pTab) || sqlite3GetVTable(db, pTab) ){ return SQLITE_OK; } @@ -698,7 +697,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ const char *zMod; pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); - assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + assert( pTab && IsVirtual(pTab) && !pTab->pVTable ); /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; @@ -752,7 +751,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ return SQLITE_MISUSE_BKPT; } pTab = pCtx->pTab; - assert( (pTab->tabFlags & TF_Virtual)!=0 ); + assert( IsVirtual(pTab) ); pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ @@ -766,7 +765,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ && pParse->pNewTable && !db->mallocFailed && !pParse->pNewTable->pSelect - && (pParse->pNewTable->tabFlags & TF_Virtual)==0 + && !IsVirtual(pParse->pNewTable) ){ if( !pTab->aCol ){ Table *pNew = pParse->pNewTable; @@ -1055,7 +1054,7 @@ FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; pTab = pExpr->pTab; if( NEVER(pTab==0) ) return pDef; - if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; + if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); assert( pVtab->pModule!=0 ); @@ -1150,7 +1149,6 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ pMod->pEpoTab = pTab; pTab->nTabRef = 1; pTab->pSchema = db->aDb[0].pSchema; - pTab->tabFlags |= TF_Virtual; pTab->nModuleArg = 0; pTab->iPKey = -1; addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); @@ -1222,7 +1220,7 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ if( !p ){ rc = SQLITE_MISUSE_BKPT; }else{ - assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + assert( p->pTab==0 || IsVirtual(p->pTab) ); p->pVTable->bConstraint = (u8)va_arg(ap, int); } break; From b6bf97b384b272285b649a031cfe150d89b2475c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 15:06:06 +0000 Subject: [PATCH 162/292] Remove two redundant initializations from the virtual table logic. FossilOrigin-Name: 6bd82b95a6b78bb60569af4da58ef4b9f997fe7b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vtab.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index db4da4c16d..f9083dc9e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\suse\sthe\sIsVirtual()\smacro\sto\sdetermine\sif\sa\sTable\sobject\sis\sa\svirtual\ntable.\s\sSlightly\ssmaller\sand\sfaster\scode. -D 2017-02-16T14:48:08.387 +C Remove\stwo\sredundant\sinitializations\sfrom\sthe\svirtual\stable\slogic. +D 2017-02-16T15:06:06.357 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -470,7 +470,7 @@ F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 -F src/vtab.c f7f26f9db3c328824674d0ed4fd9a961a9919186 +F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58797e9bafa95709e0f706a15f42f93b409e2db5 -R db3fb8708d0cf4bffde97f5d6d76493c +P 6affb1c89d87288cad87dde5a533832cdf06b8aa +R a7d037042c5db23b2b7803c8953ffd29 U drh -Z 8764b23fc3772844f5b9c915005803c6 +Z 51668c807e51ffc5d9cef025e62b59da diff --git a/manifest.uuid b/manifest.uuid index c7ccabdfe1..af3072342a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6affb1c89d87288cad87dde5a533832cdf06b8aa \ No newline at end of file +6bd82b95a6b78bb60569af4da58ef4b9f997fe7b \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index d68ec8f627..48891034d5 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -339,7 +339,7 @@ void sqlite3VtabBeginParse( iDb = sqlite3SchemaToIndex(db, pTable->pSchema); assert( iDb>=0 ); - pTable->nModuleArg = 0; + assert( pTable->nModuleArg==0 ); addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); @@ -1149,7 +1149,7 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ pMod->pEpoTab = pTab; pTab->nTabRef = 1; pTab->pSchema = db->aDb[0].pSchema; - pTab->nModuleArg = 0; + assert( pTab->nModuleArg==0 ); pTab->iPKey = -1; addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); addModuleArgument(db, pTab, 0); From 6f271a421d5def6053a2dbb3750169cdcf5ea6b0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 15:57:30 +0000 Subject: [PATCH 163/292] Increase Table.tabFlags from 8 to 32 bits. FossilOrigin-Name: 7e14044c65f64322769bcad4640a5896be0a1687 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f9083dc9e5..c8783138e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\stwo\sredundant\sinitializations\sfrom\sthe\svirtual\stable\slogic. -D 2017-02-16T15:06:06.357 +C Increase\sTable.tabFlags\sfrom\s8\sto\s32\sbits. +D 2017-02-16T15:57:30.827 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -399,7 +399,7 @@ F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h b4a3871bda47ff79b4be612a2ab89775fa737583 +F src/sqliteInt.h edd1ffa288c28d9aa316ad352a75ce1c12bcc005 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6affb1c89d87288cad87dde5a533832cdf06b8aa -R a7d037042c5db23b2b7803c8953ffd29 +P 6bd82b95a6b78bb60569af4da58ef4b9f997fe7b +R 5056d17e833da8353f5af9a38b309f26 U drh -Z 51668c807e51ffc5d9cef025e62b59da +Z 46bea1f1d9f6eb27acec6234891ad166 diff --git a/manifest.uuid b/manifest.uuid index af3072342a..890afb62ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bd82b95a6b78bb60569af4da58ef4b9f997fe7b \ No newline at end of file +7e14044c65f64322769bcad4640a5896be0a1687 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c8f42e315c..f2f2cece7a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1848,6 +1848,7 @@ struct Table { /* ... also used as column name list in a VIEW */ int tnum; /* Root BTree page for this table */ u32 nTabRef; /* Number of pointers to this Table */ + u32 tabFlags; /* Mask of TF_* values */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ @@ -1855,7 +1856,6 @@ struct Table { #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ #endif - u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ From c5f4816fc5f5716bb4f09fb429b2246145ca166a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 16:26:53 +0000 Subject: [PATCH 164/292] Fix a comment on a field of the ExprList object. No changes to code. FossilOrigin-Name: bb8e264227175fc93f1c86a0083f8ad6c4ce2dc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c8783138e0..2f0acb041e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sTable.tabFlags\sfrom\s8\sto\s32\sbits. -D 2017-02-16T15:57:30.827 +C Fix\sa\scomment\son\sa\sfield\sof\sthe\sExprList\sobject.\s\sNo\schanges\sto\scode. +D 2017-02-16T16:26:53.932 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -399,7 +399,7 @@ F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h edd1ffa288c28d9aa316ad352a75ce1c12bcc005 +F src/sqliteInt.h 46fe8e5aee3825d77fa771216ef263dc947030e7 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6bd82b95a6b78bb60569af4da58ef4b9f997fe7b -R 5056d17e833da8353f5af9a38b309f26 +P 7e14044c65f64322769bcad4640a5896be0a1687 +R 1b3160298650a3391d35f7b330db72f9 U drh -Z 46bea1f1d9f6eb27acec6234891ad166 +Z 62831885c60053730e2b44cb5ef10954 diff --git a/manifest.uuid b/manifest.uuid index 890afb62ef..878e0d699a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e14044c65f64322769bcad4640a5896be0a1687 \ No newline at end of file +bb8e264227175fc93f1c86a0083f8ad6c4ce2dc7 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f2f2cece7a..9a53d33626 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2440,7 +2440,7 @@ struct Expr { struct ExprList { int nExpr; /* Number of expressions on the list */ struct ExprList_item { /* For each expression in the list */ - Expr *pExpr; /* The list of expressions */ + Expr *pExpr; /* The parse tree for this expression */ char *zName; /* Token associated with this expression */ char *zSpan; /* Original text of the expression */ u8 sortOrder; /* 1 for DESC or 0 for ASC */ From 5c258dc1cc40a09cbfb51857fe208db60b234d64 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 17:18:07 +0000 Subject: [PATCH 165/292] Change two MallocZero() calls into MallocRaw() to avoid unnecessary memset(). FossilOrigin-Name: ff5e733cbffd73faa4046e0f1c7f24bb6e131738 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- src/legacy.c | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2f0acb041e..e0da0536b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\son\sa\sfield\sof\sthe\sExprList\sobject.\s\sNo\schanges\sto\scode. -D 2017-02-16T16:26:53.932 +C Change\stwo\sMallocZero()\scalls\sinto\sMallocRaw()\sto\savoid\sunnecessary\smemset(). +D 2017-02-16T17:18:07.363 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -347,7 +347,7 @@ F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c 38bd92fcbd86b3904bfa29e477412e374b7df5a2 +F src/expr.c 8a29e9b72d4b642189999c41782cd6c5bc43512f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -357,7 +357,7 @@ F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe -F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e +F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 F src/malloc.c d0a1474236486165bcb349af82e2a6560178bf7b @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e14044c65f64322769bcad4640a5896be0a1687 -R 1b3160298650a3391d35f7b330db72f9 +P bb8e264227175fc93f1c86a0083f8ad6c4ce2dc7 +R 99427a1cfe699c929367ef60ee07c16f U drh -Z 62831885c60053730e2b44cb5ef10954 +Z 3888dc5073b69fac4d8570651367948c diff --git a/manifest.uuid b/manifest.uuid index 878e0d699a..21b82efc27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb8e264227175fc93f1c86a0083f8ad6c4ce2dc7 \ No newline at end of file +ff5e733cbffd73faa4046e0f1c7f24bb6e131738 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 505e56a272..2bf76eb0d7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2367,7 +2367,7 @@ static char *exprINAffinity(Parse *pParse, Expr *pExpr){ char *zRet; assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocZero(pParse->db, nVal+1); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); if( zRet ){ int i; for(i=0; iflags&SQLITE_NullCallback)) ){ if( !callbackIsInit ){ - azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); if( azCols==0 ){ goto exec_out; } @@ -94,6 +94,7 @@ int sqlite3_exec( goto exec_out; } } + azVals[i] = 0; } if( xCallback(pArg, nCol, azVals, azCols) ){ /* EVIDENCE-OF: R-38229-40159 If the callback function to From e9ba910f0d358a93c95ca615b0b6c0e08fe97691 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Feb 2017 20:52:52 +0000 Subject: [PATCH 166/292] Change the name of WhereInfo.pDistinctSet to pResultSet, since it is now used for more than just DISTINCT processing. FossilOrigin-Name: 9fc5cd505fe6ab043519d68e999d2285e22452af --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 16 ++++++++-------- src/whereInt.h | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index e0da0536b9..4ce84926b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\stwo\sMallocZero()\scalls\sinto\sMallocRaw()\sto\savoid\sunnecessary\smemset(). -D 2017-02-16T17:18:07.363 +C Change\sthe\sname\sof\sWhereInfo.pDistinctSet\sto\spResultSet,\ssince\sit\sis\snow\nused\sfor\smore\sthan\sjust\sDISTINCT\sprocessing. +D 2017-02-16T20:52:52.517 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 @@ -475,8 +475,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 6397fab50fdbf9bde76c574ce07b3b776eb28b34 -F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d +F src/where.c 01baf58b72f3ddb7793cdee2871f751e3e09b35e +F src/whereInt.h c0b092180f04608d80c258174b0a14a1f9c8d02f F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb8e264227175fc93f1c86a0083f8ad6c4ce2dc7 -R 99427a1cfe699c929367ef60ee07c16f +P ff5e733cbffd73faa4046e0f1c7f24bb6e131738 +R 771ea12d7c4097594afa32049854d91e U drh -Z 3888dc5073b69fac4d8570651367948c +Z f13a633082a8173e1922e71dbb49e7e9 diff --git a/manifest.uuid b/manifest.uuid index 21b82efc27..fe9afd07d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff5e733cbffd73faa4046e0f1c7f24bb6e131738 \ No newline at end of file +9fc5cd505fe6ab043519d68e999d2285e22452af \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c533aa493..f2bf400a2d 100644 --- a/src/where.c +++ b/src/where.c @@ -4114,9 +4114,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ && nRowEst ){ Bitmask notUsed; - int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom, + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==pWInfo->pDistinctSet->nExpr ){ + if( rc==pWInfo->pResultSet->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } @@ -4353,7 +4353,7 @@ WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ - ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */ + ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -4429,7 +4429,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; - pWInfo->pDistinctSet = pDistinctSet; + pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); @@ -4507,13 +4507,13 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ /* Try to ORDER BY the result set to make distinct processing easier */ pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pDistinctSet; + pWInfo->pOrderBy = pResultSet; } } @@ -4589,10 +4589,10 @@ WhereInfo *sqlite3WhereBegin( #endif /* Attempt to omit tables from the join that do not effect the result */ if( pWInfo->nLevel>=2 - && pDistinctSet!=0 + && pResultSet!=0 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ - Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet); + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); if( sWLB.pOrderBy ){ tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); } diff --git a/src/whereInt.h b/src/whereInt.h index fd6ebe77b4..c6195f53ed 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -411,7 +411,7 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinctSet; /* DISTINCT over all these values */ + ExprList *pResultSet; /* Result set of the query */ LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ From 2b44fd94baa4f9934c646eacf1a5be56760eb146 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 01:43:51 +0000 Subject: [PATCH 167/292] Enable the ".wheretrace" and ".selecttrace" extensions in the command-line shell when compiled on Windows using DEBUG=3 or higher. Fix a harmless warning in the shell that comes up when compiled this way. FossilOrigin-Name: 8a03be1dc42737ba0712d33f639ea26dc243b20e --- Makefile.msc | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 8b98fe4cac..317f68fa9b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -745,6 +745,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF diff --git a/manifest b/manifest index 4ce84926b2..d52e7b226a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Change\sthe\sname\sof\sWhereInfo.pDistinctSet\sto\spResultSet,\ssince\sit\sis\snow\nused\sfor\smore\sthan\sjust\sDISTINCT\sprocessing. -D 2017-02-16T20:52:52.517 +C Enable\sthe\s".wheretrace"\sand\s".selecttrace"\sextensions\sin\sthe\scommand-line\nshell\swhen\scompiled\son\sWindows\susing\sDEBUG=3\sor\shigher.\s\sFix\sa\sharmless\nwarning\sin\sthe\sshell\sthat\scomes\sup\swhen\scompiled\sthis\sway. +D 2017-02-17T01:43:51.171 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 067a6766f800cc8d72845ab61f8de4ffe8f3fc99 +F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -395,7 +395,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c a661e7ccd202b16cb5321999354699e5ee018fb2 +F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff5e733cbffd73faa4046e0f1c7f24bb6e131738 -R 771ea12d7c4097594afa32049854d91e +P 9fc5cd505fe6ab043519d68e999d2285e22452af +R f90eded334f732494d0503ad2f56a8b5 U drh -Z f13a633082a8173e1922e71dbb49e7e9 +Z 4e5822b5948bc4c5b310214a2902dbfe diff --git a/manifest.uuid b/manifest.uuid index fe9afd07d4..2379d6ab79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fc5cd505fe6ab043519d68e999d2285e22452af \ No newline at end of file +8a03be1dc42737ba0712d33f639ea26dc243b20e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index fe62bfc8a1..fbbbea9d79 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4614,7 +4614,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ - sqlite3SelectTrace = integerValue(azArg[1]); + sqlite3SelectTrace = (int)integerValue(azArg[1]); }else #endif From 7898bfff26aec968ec23a46dc05947e070a83c5d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 02:04:31 +0000 Subject: [PATCH 168/292] Fix a test case that was made to fail by the LIKE optimization enhancement in check-in [158290c0ab] but which went unnoticed because test builds were running with ICU enabled and ICU disables the LIKE optimization. FossilOrigin-Name: 218b2bbb0de07288889f6762d4461ea8acd78969 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/tkt-78e04e52ea.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d52e7b226a..dc0eaa5c63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\s".wheretrace"\sand\s".selecttrace"\sextensions\sin\sthe\scommand-line\nshell\swhen\scompiled\son\sWindows\susing\sDEBUG=3\sor\shigher.\s\sFix\sa\sharmless\nwarning\sin\sthe\sshell\sthat\scomes\sup\swhen\scompiled\sthis\sway. -D 2017-02-17T01:43:51.171 +C Fix\sa\stest\scase\sthat\swas\smade\sto\sfail\sby\sthe\sLIKE\soptimization\senhancement\nin\scheck-in\s[158290c0ab]\sbut\swhich\swent\sunnoticed\sbecause\stest\sbuilds\swere\nrunning\swith\sICU\senabled\sand\sICU\sdisables\sthe\sLIKE\soptimization. +D 2017-02-17T02:04:31.068 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -1206,7 +1206,7 @@ F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf -F test/tkt-78e04e52ea.test 813779f8888f3ca226df656c4eef078f9635f3c9 +F test/tkt-78e04e52ea.test 1b2e6bf4f1d9887b216b6da774e5f25915ec8118 F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9fc5cd505fe6ab043519d68e999d2285e22452af -R f90eded334f732494d0503ad2f56a8b5 +P 8a03be1dc42737ba0712d33f639ea26dc243b20e +R b29803fdd4fd514ba7baa53a58b26311 U drh -Z 4e5822b5948bc4c5b310214a2902dbfe +Z af65459984ca8c135fabce3c62812860 diff --git a/manifest.uuid b/manifest.uuid index 2379d6ab79..d642d67abe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a03be1dc42737ba0712d33f639ea26dc243b20e \ No newline at end of file +218b2bbb0de07288889f6762d4461ea8acd78969 \ No newline at end of file diff --git a/test/tkt-78e04e52ea.test b/test/tkt-78e04e52ea.test index 975e5b3d1c..963ecf18d1 100644 --- a/test/tkt-78e04e52ea.test +++ b/test/tkt-78e04e52ea.test @@ -42,7 +42,7 @@ do_test tkt-78e04-1.3 { } {} do_test tkt-78e04-1.4 { execsql { - EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE 'abc%'; + EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE '1abc%'; } } {0 0 0 {SCAN TABLE USING COVERING INDEX i1}} do_test tkt-78e04-1.5 { From 33bec3f5e89787125b56243ab7de2fe9353b8b8a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 13:38:15 +0000 Subject: [PATCH 169/292] Enhance the Index and Table objects so that they remember if their stats come from the sqlite_stat1 table. Make the "PRAGMA stats" an SQLITE_DEBUG only pragma. Add the flags column to "PRAGMA stats". These are all preliminary steps toward a "PRAGMA analyze_ifneeded;" feature. FossilOrigin-Name: 85026c8ee143bbd46565660fff8346ef81421546 --- manifest | 25 +++++---- manifest.uuid | 2 +- src/analyze.c | 22 ++++++-- src/build.c | 3 + src/pragma.c | 18 +++--- src/pragma.h | 129 ++++++++++++++++++++++--------------------- src/sqliteInt.h | 18 +++--- tool/mkpragmatab.tcl | 4 +- 8 files changed, 121 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index dc0eaa5c63..04051afae1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sthat\swas\smade\sto\sfail\sby\sthe\sLIKE\soptimization\senhancement\nin\scheck-in\s[158290c0ab]\sbut\swhich\swent\sunnoticed\sbecause\stest\sbuilds\swere\nrunning\swith\sICU\senabled\sand\sICU\sdisables\sthe\sLIKE\soptimization. -D 2017-02-17T02:04:31.068 +C Enhance\sthe\sIndex\sand\sTable\sobjects\sso\sthat\sthey\sremember\sif\stheir\sstats\scome\nfrom\sthe\ssqlite_stat1\stable.\s\sMake\sthe\s"PRAGMA\sstats"\san\sSQLITE_DEBUG\sonly\npragma.\s\sAdd\sthe\sflags\scolumn\sto\s"PRAGMA\sstats".\s\sThese\sare\sall\spreliminary\nsteps\stoward\sa\s"PRAGMA\sanalyze_ifneeded;"\sfeature. +D 2017-02-17T13:38:15.256 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c ac7a5d7e3cee07d074697904e00e4a8ab7b2b4f5 +F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -340,7 +340,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 -F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af +F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 @@ -387,8 +387,8 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d -F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a +F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 +F src/pragma.h cea24a631982fd1a26fcddd46f596d22303b4247 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 46fe8e5aee3825d77fa771216ef263dc947030e7 +F src/sqliteInt.h 54bc20855f2a1a99a1b726f1384b5ce9ab67a0ff F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1494,7 +1494,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl ebb4bfcd2f8010e0a3934b6118db4b5f2f5edf5c +F tool/mkpragmatab.tcl 9c0a855e0daf83e54ffd9fd438260cd0c92f25d0 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1556,7 +1556,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a03be1dc42737ba0712d33f639ea26dc243b20e -R b29803fdd4fd514ba7baa53a58b26311 +P 218b2bbb0de07288889f6762d4461ea8acd78969 +R af8aeffdf276070e8e8912906300a1a4 +T *branch * auto-analyze +T *sym-auto-analyze * +T -sym-trunk * U drh -Z af65459984ca8c135fabce3c62812860 +Z f8720207819c88fb362c87de069d287a diff --git a/manifest.uuid b/manifest.uuid index d642d67abe..fd9de00f55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -218b2bbb0de07288889f6762d4461ea8acd78969 \ No newline at end of file +85026c8ee143bbd46565660fff8346ef81421546 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 890ae7c3b2..3ecf469cd3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1532,7 +1532,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -1541,6 +1545,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -1835,15 +1840,20 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ HashElem *i; char *zSql; int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; + pIdx->hasStat1 = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; @@ -1866,9 +1876,9 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); } /* Load the statistics from the sqlite_stat4 table. */ @@ -1878,7 +1888,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; diff --git a/src/build.c b/src/build.c index cd9c81be82..f85c55f0a1 100644 --- a/src/build.c +++ b/src/build.c @@ -3454,6 +3454,9 @@ void sqlite3DefaultRowEst(Index *pIdx){ int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + /* Set the first entry (number of rows in the index) to the estimated ** number of rows in the table, or half the number of rows in the table ** for a partial index. But do not let the estimate drop below 10. */ diff --git a/src/pragma.c b/src/pragma.c index b1775a4082..f28ff9d13c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1098,29 +1098,33 @@ void sqlite3Pragma( } break; +#ifdef SQLITE_DEBUG case PragTyp_STATS: { Index *pIdx; HashElem *i; - pParse->nMem = 4; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", + sqlite3VdbeMultiLoad(v, 1, "ssiii", pTab->zName, 0, pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pTab->nRowLogEst, + pTab->tabFlags); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", + sqlite3VdbeMultiLoad(v, 2, "siii", pIdx->zName, pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; +#endif case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; diff --git a/src/pragma.h b/src/pragma.h index 5d8d0aa35b..2a96ecaa43 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -34,20 +34,20 @@ #define PragTyp_SECURE_DELETE 26 #define PragTyp_SHRINK_MEMORY 27 #define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 +#define PragTyp_STATS 42 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -75,43 +75,44 @@ static const char *const pragCName[] = { /* 8 */ "index", /* 9 */ "width", /* 10 */ "height", - /* 11 */ "seqno", /* Used by: index_info */ - /* 12 */ "cid", - /* 13 */ "name", - /* 14 */ "seqno", /* Used by: index_xinfo */ - /* 15 */ "cid", - /* 16 */ "name", - /* 17 */ "desc", - /* 18 */ "coll", - /* 19 */ "key", - /* 20 */ "seq", /* Used by: index_list */ - /* 21 */ "name", - /* 22 */ "unique", - /* 23 */ "origin", - /* 24 */ "partial", - /* 25 */ "seq", /* Used by: database_list */ - /* 26 */ "name", - /* 27 */ "file", - /* 28 */ "seq", /* Used by: collation_list */ - /* 29 */ "name", - /* 30 */ "id", /* Used by: foreign_key_list */ - /* 31 */ "seq", - /* 32 */ "table", - /* 33 */ "from", - /* 34 */ "to", - /* 35 */ "on_update", - /* 36 */ "on_delete", - /* 37 */ "match", - /* 38 */ "table", /* Used by: foreign_key_check */ - /* 39 */ "rowid", - /* 40 */ "parent", - /* 41 */ "fkid", - /* 42 */ "busy", /* Used by: wal_checkpoint */ - /* 43 */ "log", - /* 44 */ "checkpointed", - /* 45 */ "timeout", /* Used by: busy_timeout */ - /* 46 */ "database", /* Used by: lock_status */ - /* 47 */ "status", + /* 11 */ "flags", + /* 12 */ "seqno", /* Used by: index_info */ + /* 13 */ "cid", + /* 14 */ "name", + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "seq", /* Used by: index_list */ + /* 22 */ "name", + /* 23 */ "unique", + /* 24 */ "origin", + /* 25 */ "partial", + /* 26 */ "seq", /* Used by: database_list */ + /* 27 */ "name", + /* 28 */ "file", + /* 29 */ "seq", /* Used by: collation_list */ + /* 30 */ "name", + /* 31 */ "id", /* Used by: foreign_key_list */ + /* 32 */ "seq", + /* 33 */ "table", + /* 34 */ "from", + /* 35 */ "to", + /* 36 */ "on_update", + /* 37 */ "on_delete", + /* 38 */ "match", + /* 39 */ "table", /* Used by: foreign_key_check */ + /* 40 */ "rowid", + /* 41 */ "parent", + /* 42 */ "fkid", + /* 43 */ "busy", /* Used by: wal_checkpoint */ + /* 44 */ "log", + /* 45 */ "checkpointed", + /* 46 */ "timeout", /* Used by: busy_timeout */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", }; /* Definitions of all built-in pragmas */ @@ -157,7 +158,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 45, 1, + /* ColNames: */ 46, 1, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "cache_size", @@ -194,7 +195,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 28, 2, + /* ColNames: */ 29, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -229,7 +230,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 25, 3, + /* ColNames: */ 26, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -266,14 +267,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "foreign_key_check", /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 38, 4, + /* ColNames: */ 39, 4, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) {/* zName: */ "foreign_key_list", /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 30, 8, + /* ColNames: */ 31, 8, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -336,17 +337,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 11, 3, + /* ColNames: */ 12, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 20, 5, + /* ColNames: */ 21, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 14, 6, + /* ColNames: */ 15, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -393,7 +394,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "lock_status", /* ePragTyp: */ PragTyp_LOCK_STATUS, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 46, 2, + /* ColNames: */ 47, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -510,11 +511,11 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 7, 4, + /* ColNames: */ 7, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -593,7 +594,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 42, 3, + /* ColNames: */ 43, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -604,4 +605,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 59 on by default, 73 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9a53d33626..6d58157b1b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1879,15 +1879,14 @@ struct Table { ** the TF_OOOHidden attribute would apply in this case. Such tables require ** special handling during INSERT processing. */ -#define TF_Readonly 0x01 /* Read-only system table */ -#define TF_Ephemeral 0x02 /* An ephemeral table */ -#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ -#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -/* available for reuse: 0x10 */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ - +#define TF_Readonly 0x0001 /* Read-only system table */ +#define TF_Ephemeral 0x0002 /* An ephemeral table */ +#define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ +#define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ +#define TF_HasStat1 0x0010 /* Has STAT1 data */ +#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2130,6 +2129,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* True if stat1 data has been loaded */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index c22f72d491..1c8bad54d3 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,8 +225,8 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height - IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + COLS: table index width height flags + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info TYPE: INDEX_INFO From a3928dd7bee435f2cf4288b336968f6334376d39 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 15:26:36 +0000 Subject: [PATCH 170/292] Set the TF_StatsUsed flag on tables when the query planner outcome is affected by the sqlite_stat1 data. Also, change the column names of the "PRAGMA stats" command so that they are not keywords. FossilOrigin-Name: fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 --- manifest | 24 ++++---- manifest.uuid | 2 +- src/pragma.h | 10 ++-- src/sqliteInt.h | 7 ++- src/where.c | 13 +++++ src/whereInt.h | 5 ++ test/autoanalyze1.test | 124 +++++++++++++++++++++++++++++++++++++++++ tool/mkpragmatab.tcl | 2 +- 8 files changed, 165 insertions(+), 22 deletions(-) create mode 100644 test/autoanalyze1.test diff --git a/manifest b/manifest index 04051afae1..360ecf7903 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sIndex\sand\sTable\sobjects\sso\sthat\sthey\sremember\sif\stheir\sstats\scome\nfrom\sthe\ssqlite_stat1\stable.\s\sMake\sthe\s"PRAGMA\sstats"\san\sSQLITE_DEBUG\sonly\npragma.\s\sAdd\sthe\sflags\scolumn\sto\s"PRAGMA\sstats".\s\sThese\sare\sall\spreliminary\nsteps\stoward\sa\s"PRAGMA\sanalyze_ifneeded;"\sfeature. -D 2017-02-17T13:38:15.256 +C Set\sthe\sTF_StatsUsed\sflag\son\stables\swhen\sthe\squery\splanner\soutcome\sis\naffected\sby\sthe\ssqlite_stat1\sdata.\s\sAlso,\schange\sthe\scolumn\snames\sof\sthe\n"PRAGMA\sstats"\scommand\sso\sthat\sthey\sare\snot\skeywords. +D 2017-02-17T15:26:36.765 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,7 +388,7 @@ F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 -F src/pragma.h cea24a631982fd1a26fcddd46f596d22303b4247 +F src/pragma.h 9e65a9033cce7387c27cd89aecb2b2f205d1edf7 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 54bc20855f2a1a99a1b726f1384b5ce9ab67a0ff +F src/sqliteInt.h 5d50606deed2b38b35fb1b5e4ab658f8faa37d4a F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -475,8 +475,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 01baf58b72f3ddb7793cdee2871f751e3e09b35e -F src/whereInt.h c0b092180f04608d80c258174b0a14a1f9c8d02f +F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -521,6 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1494,7 +1495,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 9c0a855e0daf83e54ffd9fd438260cd0c92f25d0 +F tool/mkpragmatab.tcl c955db934f7b1d800081447042cef692f26b2516 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1556,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 218b2bbb0de07288889f6762d4461ea8acd78969 -R af8aeffdf276070e8e8912906300a1a4 -T *branch * auto-analyze -T *sym-auto-analyze * -T -sym-trunk * +P 85026c8ee143bbd46565660fff8346ef81421546 +R 7cc0cc258beb93b198b99dc8e7b58520 U drh -Z f8720207819c88fb362c87de069d287a +Z 76eea0dfcfb8fa07347b6cdca98e2297 diff --git a/manifest.uuid b/manifest.uuid index fd9de00f55..1142241825 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85026c8ee143bbd46565660fff8346ef81421546 \ No newline at end of file +fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 \ No newline at end of file diff --git a/src/pragma.h b/src/pragma.h index 2a96ecaa43..1592bbd2d5 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -71,11 +71,11 @@ static const char *const pragCName[] = { /* 4 */ "notnull", /* 5 */ "dflt_value", /* 6 */ "pk", - /* 7 */ "table", /* Used by: stats */ - /* 8 */ "index", - /* 9 */ "width", - /* 10 */ "height", - /* 11 */ "flags", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", /* 12 */ "seqno", /* Used by: index_info */ /* 13 */ "cid", /* 14 */ "name", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6d58157b1b..3c75f2ff6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1883,10 +1883,13 @@ struct Table { #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ -#define TF_HasStat1 0x0010 /* Has STAT1 data */ +#define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_SizeChng 0x0100 /* nRowLogEst might be inaccurate */ +#define TF_StatsUsed 0x0200 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2129,7 +2132,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ - unsigned hasStat1:1; /* True if stat1 data has been loaded */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/where.c b/src/where.c index f2bf400a2d..8d0dbb0979 100644 --- a/src/where.c +++ b/src/where.c @@ -2387,6 +2387,11 @@ static int whereLoopAddBtreeIndex( continue; } + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + }else{ + pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->u.btree.nBtm = saved_nBtm; @@ -2934,7 +2939,15 @@ static int whereLoopAddBtree( } } + pBuilder->bldFlags = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; diff --git a/src/whereInt.h b/src/whereInt.h index c6195f53ed..f065fae6ba 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -395,8 +395,13 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned int bldFlags; /* SQLITE_BLDF_* flags */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test new file mode 100644 index 0000000000..c36e0fcbc7 --- /dev/null +++ b/test/autoanalyze1.test @@ -0,0 +1,124 @@ +# 2017-02-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for the logic used to estimate when +# running ANALYZE would be beneficial. +# +# Note that this test uses some hard-coded bitmask values from sqliteInt.h. +# If any of the following constants changes: +# +# define TF_HasStat1 0x0010 +# define TF_SizeChng 0x0100 +# define TF_StatsUsed 0x0200 +# +# then some of the magic numbers in test results below might need to be +# adjusted. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# There is nothing to test if ANALYZE is disable for this build. +# These tests also use "PRAGMA stats" which are only enabled for +# debugging builds. +# +ifcapable {!debug || !analyze} { + finish_test + return +} + +do_execsql_test autoanalyze1-100 { + -- Build up a test table with some indexes + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE UNIQUE INDEX t1bc ON t1(b,c); + CREATE INDEX t1d ON t1(d); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b,c,d) SELECT x, x, x, x FROM c; + -- Verify that the hasStat1 flag is clear on on indexes + SELECT idx, flgs FROM pragma_stats + WHERE idx IS NOT NULL + ORDER BY idx; + -- Verify that the TF_HasStat1 flag is clear on the table + SELECT tbl, (flgs & 0x10)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {t1bc 0 t1d 0 t1 0} + +# No use of stat1 recorded so far +do_execsql_test autoanalyze1-110 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Access using a unique index does not set the TF_StatsUsed flag. +# +do_execsql_test autoanalyze1-200 { + SELECT * FROM t1 WHERE a=55; +} {55 55 55 55} +do_execsql_test autoanalyze1-201 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-210 { + SELECT * FROM t1 WHERE a IN (55,199,299); +} {55 55 55 55} +do_execsql_test autoanalyze1-211 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-220 { + SELECT * FROM t1 WHERE (b,c)=(45,45); +} {45 45 45 45} +do_execsql_test autoanalyze1-221 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Any use of the non-unique t1d index triggers the use of stats. +# +sqlite3 db test.db +do_execsql_test autoanalyze1-300 { + SELECT * FROM t1 WHERE d=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-301 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +sqlite3 db test.db +do_execsql_test autoanalyze1-310 { + SELECT * FROM t1 WHERE d=45 AND a=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-311 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} ;# The ROWID lookup short-circuits the d=45 constraint + +sqlite3 db test.db +do_execsql_test autoanalyze1-320 { + SELECT * FROM t1 WHERE d=45 AND a IN (45,46); +} {45 45 45 45} +do_execsql_test autoanalyze1-321 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# Any use of prefix of a unique index triggers the use of stats +# +sqlite3 db test.db +do_execsql_test autoanalyze1-400 { + SELECT * FROM t1 WHERE b=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-401 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# The TF_StatsUsed flag is reset when the database is reopened +# +sqlite3 db test.db +do_execsql_test autoanalyze1-500 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +finish_test diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 1c8bad54d3..005c4bc48e 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,7 +225,7 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height flags + COLS: tbl idx wdth hght flgs IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info From 72052a73a3d1fc75015aa4123bea9af2f5a58e3f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 16:26:34 +0000 Subject: [PATCH 171/292] Add the "PRAGMA analyze_as_needed" command. FossilOrigin-Name: e93db2373127d31d33ec46ef918fa9386bb664a6 --- manifest | 22 +++++------ manifest.uuid | 2 +- src/analyze.c | 88 ++++++++++++++++++++++++++++++----------- src/parse.y | 4 +- src/pragma.c | 8 ++++ src/pragma.h | 94 +++++++++++++++++++++++--------------------- src/sqliteInt.h | 2 +- tool/mkpragmatab.tcl | 3 ++ 8 files changed, 141 insertions(+), 82 deletions(-) diff --git a/manifest b/manifest index 360ecf7903..9d27e603a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sthe\sTF_StatsUsed\sflag\son\stables\swhen\sthe\squery\splanner\soutcome\sis\naffected\sby\sthe\ssqlite_stat1\sdata.\s\sAlso,\schange\sthe\scolumn\snames\sof\sthe\n"PRAGMA\sstats"\scommand\sso\sthat\sthey\sare\snot\skeywords. -D 2017-02-17T15:26:36.765 +C Add\sthe\s"PRAGMA\sanalyze_as_needed"\scommand. +D 2017-02-17T16:26:34.780 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac +F src/analyze.c 39206f31ca136a0a4d4f1d6bfd00bc81c6b3e5ec F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -383,12 +383,12 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c +F src/parse.y d5695ae4887a52352b1b8a3e3623477c7e79abeb F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 -F src/pragma.h 9e65a9033cce7387c27cd89aecb2b2f205d1edf7 +F src/pragma.c e8b2ea66dfb1b90c53b920c08914ac26b109053a +F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 5d50606deed2b38b35fb1b5e4ab658f8faa37d4a +F src/sqliteInt.h 85706e0963964336270590a1e508dec78540bc5c F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1495,7 +1495,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl c955db934f7b1d800081447042cef692f26b2516 +F tool/mkpragmatab.tcl 6fd5ecb4a8debee39e3bb4e63f0634f69edfb861 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85026c8ee143bbd46565660fff8346ef81421546 -R 7cc0cc258beb93b198b99dc8e7b58520 +P fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 +R 23ff1c8d68460fd739a12857d6287bf8 U drh -Z 76eea0dfcfb8fa07347b6cdca98e2297 +Z 4884503796ef3cb7f91c7100ffa75ea8 diff --git a/manifest.uuid b/manifest.uuid index 1142241825..f532ee84dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 \ No newline at end of file +e93db2373127d31d33ec46ef918fa9386bb664a6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 3ecf469cd3..34de7fc744 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1286,28 +1286,66 @@ static void loadAnalysis(Parse *pParse, int iDb){ } /* -** Generate code that will do an analysis of an entire database +** Return true if table pTab is in need of being reanalyzed. */ -static void analyzeDatabase(Parse *pParse, int iDb){ +static int analyzeNeeded(Table *pTab){ + Index *pIdx; + if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; + if( (pTab->tabFlags & TF_SizeChng)!=0 ) return 1; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ) return 1; + } + return 0; +} + +/* +** Generate code that will do an analysis of an entire database. +** +** Return a count of the number of tables actually analyzed. Return 0 +** if nothing was analyzed. +*/ +static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ sqlite3 *db = pParse->db; - Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + Schema *pSchema; HashElem *k; int iStatCur; int iMem; int iTab; + int iDb; /* Database currently being analyzed */ + int iDbFirst, iDbLast; /* Range of databases to be analyzed */ + int cnt; /* Number of tables analyzed in a single database */ + int allCnt = 0; /* Number of tables analyzed across all databases */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab; - pParse->nTab += 3; - openStatTable(pParse, iDb, iStatCur, 0, 0); - iMem = pParse->nMem+1; - iTab = pParse->nTab; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ - Table *pTab = (Table*)sqliteHashData(k); - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + if( iDbReq>=0 ){ + iDbFirst = iDbLast = iDbReq; + }else{ + iDbFirst = 0; + iDbLast = db->nDb-1; } - loadAnalysis(pParse, iDb); + for(iDb=iDbFirst; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; /* Do not analyze the TEMP database */ + pSchema = db->aDb[iDb].pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + cnt = 0; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + if( !onlyIfNeeded || analyzeNeeded(pTab) ){ + if( cnt==0 ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + } + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + cnt++; + allCnt++; + } + } + if( cnt ) loadAnalysis(pParse, iDb); + } + return allCnt; } /* @@ -1345,16 +1383,21 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** Form 1 causes all indices in all attached databases to be analyzed. ** Form 2 analyzes all indices the single database named. ** Form 3 analyzes all indices associated with the named table. +** +** If pName1 and pName2 are both NULL and if the ifNeeded flag is true, +** this routine computes an conditional ANALYZE on only those tables +** are believed to be in need of analysis. The conditional analysis +** might well be a no-op. */ -void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ +void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ sqlite3 *db = pParse->db; int iDb; - int i; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; Vdbe *v; + int needExpire = 1; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1366,15 +1409,12 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - for(i=0; inDb; i++){ - if( i==1 ) continue; /* Do not analyze the TEMP database */ - analyzeDatabase(pParse, i); - } + needExpire = analyzeDatabase(pParse, -1, ifNeeded) || ifNeeded==0; }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); + analyzeDatabase(pParse, iDb, 0); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1402,8 +1442,10 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); + if( needExpire ){ + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); + } } /* diff --git a/src/parse.y b/src/parse.y index 9cada2a1be..ff03c128ac 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1512,8 +1512,8 @@ cmd ::= REINDEX nm(X) dbnm(Y). {sqlite3Reindex(pParse, &X, &Y);} /////////////////////////////////// ANALYZE /////////////////////////////////// %ifndef SQLITE_OMIT_ANALYZE -cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0);} -cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} +cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0, 0);} +cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y, 0);} %endif //////////////////////// ALTER TABLE table ... //////////////////////////////// diff --git a/src/pragma.c b/src/pragma.c index f28ff9d13c..789751958c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1828,6 +1828,14 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA analyze_as_needed + */ + case PragTyp_ANALYZE_AS_NEEDED: { + sqlite3Analyze(pParse, 0, 0, 1); + break; + } + /* ** PRAGMA busy_timeout ** PRAGMA busy_timeout = N diff --git a/src/pragma.h b/src/pragma.h index 1592bbd2d5..ba107ec07e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -5,49 +5,50 @@ */ /* The various pragma types */ -#define PragTyp_HEADER_VALUE 0 -#define PragTyp_AUTO_VACUUM 1 -#define PragTyp_FLAG 2 -#define PragTyp_BUSY_TIMEOUT 3 -#define PragTyp_CACHE_SIZE 4 -#define PragTyp_CACHE_SPILL 5 -#define PragTyp_CASE_SENSITIVE_LIKE 6 -#define PragTyp_COLLATION_LIST 7 -#define PragTyp_COMPILE_OPTIONS 8 -#define PragTyp_DATA_STORE_DIRECTORY 9 -#define PragTyp_DATABASE_LIST 10 -#define PragTyp_DEFAULT_CACHE_SIZE 11 -#define PragTyp_ENCODING 12 -#define PragTyp_FOREIGN_KEY_CHECK 13 -#define PragTyp_FOREIGN_KEY_LIST 14 -#define PragTyp_INCREMENTAL_VACUUM 15 -#define PragTyp_INDEX_INFO 16 -#define PragTyp_INDEX_LIST 17 -#define PragTyp_INTEGRITY_CHECK 18 -#define PragTyp_JOURNAL_MODE 19 -#define PragTyp_JOURNAL_SIZE_LIMIT 20 -#define PragTyp_LOCK_PROXY_FILE 21 -#define PragTyp_LOCKING_MODE 22 -#define PragTyp_PAGE_COUNT 23 -#define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_SYNCHRONOUS 29 -#define PragTyp_TABLE_INFO 30 -#define PragTyp_TEMP_STORE 31 -#define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_THREADS 33 -#define PragTyp_WAL_AUTOCHECKPOINT 34 -#define PragTyp_WAL_CHECKPOINT 35 -#define PragTyp_ACTIVATE_EXTENSIONS 36 -#define PragTyp_HEXKEY 37 -#define PragTyp_KEY 38 -#define PragTyp_REKEY 39 -#define PragTyp_LOCK_STATUS 40 -#define PragTyp_PARSER_TRACE 41 -#define PragTyp_STATS 42 +#define PragTyp_ANALYZE_AS_NEEDED 0 +#define PragTyp_HEADER_VALUE 1 +#define PragTyp_AUTO_VACUUM 2 +#define PragTyp_FLAG 3 +#define PragTyp_BUSY_TIMEOUT 4 +#define PragTyp_CACHE_SIZE 5 +#define PragTyp_CACHE_SPILL 6 +#define PragTyp_CASE_SENSITIVE_LIKE 7 +#define PragTyp_COLLATION_LIST 8 +#define PragTyp_COMPILE_OPTIONS 9 +#define PragTyp_DATA_STORE_DIRECTORY 10 +#define PragTyp_DATABASE_LIST 11 +#define PragTyp_DEFAULT_CACHE_SIZE 12 +#define PragTyp_ENCODING 13 +#define PragTyp_FOREIGN_KEY_CHECK 14 +#define PragTyp_FOREIGN_KEY_LIST 15 +#define PragTyp_INCREMENTAL_VACUUM 16 +#define PragTyp_INDEX_INFO 17 +#define PragTyp_INDEX_LIST 18 +#define PragTyp_INTEGRITY_CHECK 19 +#define PragTyp_JOURNAL_MODE 20 +#define PragTyp_JOURNAL_SIZE_LIMIT 21 +#define PragTyp_LOCK_PROXY_FILE 22 +#define PragTyp_LOCKING_MODE 23 +#define PragTyp_PAGE_COUNT 24 +#define PragTyp_MMAP_SIZE 25 +#define PragTyp_PAGE_SIZE 26 +#define PragTyp_SECURE_DELETE 27 +#define PragTyp_SHRINK_MEMORY 28 +#define PragTyp_SOFT_HEAP_LIMIT 29 +#define PragTyp_SYNCHRONOUS 30 +#define PragTyp_TABLE_INFO 31 +#define PragTyp_TEMP_STORE 32 +#define PragTyp_TEMP_STORE_DIRECTORY 33 +#define PragTyp_THREADS 34 +#define PragTyp_WAL_AUTOCHECKPOINT 35 +#define PragTyp_WAL_CHECKPOINT 36 +#define PragTyp_ACTIVATE_EXTENSIONS 37 +#define PragTyp_HEXKEY 38 +#define PragTyp_KEY 39 +#define PragTyp_REKEY 40 +#define PragTyp_LOCK_STATUS 41 +#define PragTyp_PARSER_TRACE 42 +#define PragTyp_STATS 43 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -132,6 +133,11 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif + {/* zName: */ "analyze_as_needed", + /* ePragTyp: */ PragTyp_ANALYZE_AS_NEEDED, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -605,4 +611,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 73 total. */ +/* Number of pragmas: 60 on by default, 74 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3c75f2ff6e..f590c85f93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4028,7 +4028,7 @@ void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); -void sqlite3Analyze(Parse*, Token*, Token*); +void sqlite3Analyze(Parse*, Token*, Token*, int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 005c4bc48e..2201ea1306 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -361,6 +361,9 @@ set pragma_def { NAME: threads FLAG: Result0 + + NAME: analyze_as_needed + FLAG: NoColumns } # Open the output file From 5e98e838da57d5d70c1b3321c73e82206f5c8457 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 19:24:06 +0000 Subject: [PATCH 172/292] The analyze_as_needed pragma now responds to table size growth and will automatically rerun the analysis after each 10x size increase. FossilOrigin-Name: bfbdd07409688fac4ccddbab3639745f6152e23d --- manifest | 22 ++++++++++----------- manifest.uuid | 2 +- src/analyze.c | 43 +++++++++++++++++++++++++++++++++--------- src/btree.c | 19 +++++++++++++++++++ src/btree.h | 1 + src/sqliteInt.h | 3 +-- src/vdbe.c | 27 ++++++++++++++++++++++++++ test/autoanalyze1.test | 21 ++++++++++----------- 8 files changed, 104 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 9d27e603a6..42a11a725c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"PRAGMA\sanalyze_as_needed"\scommand. -D 2017-02-17T16:26:34.780 +C The\sanalyze_as_needed\spragma\snow\sresponds\sto\stable\ssize\sgrowth\sand\swill\nautomatically\srerun\sthe\sanalysis\safter\seach\s10x\ssize\sincrease. +D 2017-02-17T19:24:06.618 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,14 +331,14 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 39206f31ca136a0a4d4f1d6bfd00bc81c6b3e5ec +F src/analyze.c acef9c48eac30438e82f39eb8df6682c6e33942e F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 -F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac +F src/btree.c e7d724b02365d88920171caf0c919aa96d9b19e3 +F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 85706e0963964336270590a1e508dec78540bc5c +F src/sqliteInt.h 60c38b84a796cc9d0a9a794cca254b09898b9cc5 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b +F src/vdbe.c ac8538b96af5c63d6f18484ffd981d4f44db0073 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -521,7 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f -F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76 +F test/autoanalyze1.test 1ba80d5e1412d46503566b70741a5eea060da929 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 -R 23ff1c8d68460fd739a12857d6287bf8 +P e93db2373127d31d33ec46ef918fa9386bb664a6 +R 484e61dbf95be1098adb3acfa069e723 U drh -Z 4884503796ef3cb7f91c7100ffa75ea8 +Z 81466028927592a195c5763d3460fcde diff --git a/manifest.uuid b/manifest.uuid index f532ee84dd..8be7eeb5b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93db2373127d31d33ec46ef918fa9386bb664a6 \ No newline at end of file +bfbdd07409688fac4ccddbab3639745f6152e23d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 34de7fc744..082713fd23 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -958,10 +958,12 @@ static void analyzeOneTable( Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ int iMem, /* Available memory locations begin here */ - int iTab /* Next available cursor */ + int iTab, /* Next available cursor */ + LogEst szOld /* Run the analysis if table row count is larger than this */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ + int addrSizeCk = 0; /* Address of the IfSmaller */ int iIdxCur; /* Cursor open on index being analyzed */ int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ @@ -1014,6 +1016,9 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szOld>0 ){ + addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); + } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1271,6 +1276,7 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } + sqlite3VdbeJumpHere(v, addrSizeCk); } @@ -1286,16 +1292,33 @@ static void loadAnalysis(Parse *pParse, int iDb){ } /* -** Return true if table pTab is in need of being reanalyzed. +** Return true if table pTab might need to being reanalyzed. Return +** false if we know that pTab should not be reanalyzed. +** +** If returning true, also set *pThreshold to a size threshold that +** will determine at run-time whether or not the reanalysis occurs. +** The reanalysis will only occur if the size of the table is greater +** than the threshold. Not that the threshold is a logarithmic LogEst +** value. */ -static int analyzeNeeded(Table *pTab){ +static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ Index *pIdx; if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - if( (pTab->tabFlags & TF_SizeChng)!=0 ) return 1; + + /* If TF_StatsUsed is true, then we might need to reanalyze. But + ** only reanalyze if the table size has grown by a factor of 10 or more */ + *pThreshold = pTab->nRowLogEst + 33; assert( sqlite3LogEst(10)==33 ); + + /* Except, if any of the indexes of the table do not have valid + ** sqlite_stat1 entries, then set the size threshold to zero to + ** ensure the analysis will always occur. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 ) return 1; + if( !pIdx->hasStat1 ){ + *pThreshold = 0; + break; + } } - return 0; + return 1; } /* @@ -1329,7 +1352,8 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ cnt = 0; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); - if( !onlyIfNeeded || analyzeNeeded(pTab) ){ + LogEst szThreshold = 0; + if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ if( cnt==0 ){ sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; @@ -1338,7 +1362,7 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ iMem = pParse->nMem+1; iTab = pParse->nTab; } - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); cnt++; allCnt++; } @@ -1368,7 +1392,8 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1, + pParse->nTab, 0); loadAnalysis(pParse, iDb); } diff --git a/src/btree.c b/src/btree.c index e78ffef1be..8aed08cb36 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5319,6 +5319,25 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( pCur->eState!=CURSOR_VALID ) return -1; + if( pCur->apPage[pCur->iPage-1]->leaf==0 ) return -1; + for(n=1, i=0; iiPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor diff --git a/src/btree.h b/src/btree.h index ae57468e3f..0e017f5300 100644 --- a/src/btree.h +++ b/src/btree.h @@ -296,6 +296,7 @@ u32 sqlite3BtreePayloadSize(BtCursor*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f590c85f93..d9cad11bcf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1887,8 +1887,7 @@ struct Table { #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ -#define TF_SizeChng 0x0100 /* nRowLogEst might be inaccurate */ -#define TF_StatsUsed 0x0200 /* Query planner decisions affected by +#define TF_StatsUsed 0x0100 /* Query planner decisions affected by ** Index.aiRowLogEst[] values */ /* diff --git a/src/vdbe.c b/src/vdbe.c index 7f286b2c6d..b595c4fb01 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4847,6 +4847,33 @@ case OP_Last: { /* jump */ break; } +/* Opcode: IfSmaller P1 P2 P3 * * +** +** Estimate the number of rows in the table P1. Jump to P2 if that +** estimate is less than approximately 2**(0.1*P3). +*/ +case OP_IfSmaller: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + sz = sqlite3BtreeRowCountEst(pCrsr); + if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + } + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: SorterSort P1 P2 * * * ** diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test index c36e0fcbc7..77e767a89e 100644 --- a/test/autoanalyze1.test +++ b/test/autoanalyze1.test @@ -16,8 +16,7 @@ # If any of the following constants changes: # # define TF_HasStat1 0x0010 -# define TF_SizeChng 0x0100 -# define TF_StatsUsed 0x0200 +# define TF_StatsUsed 0x0100 # # then some of the magic numbers in test results below might need to be # adjusted. @@ -52,7 +51,7 @@ do_execsql_test autoanalyze1-100 { # No use of stat1 recorded so far do_execsql_test autoanalyze1-110 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} # Access using a unique index does not set the TF_StatsUsed flag. @@ -61,21 +60,21 @@ do_execsql_test autoanalyze1-200 { SELECT * FROM t1 WHERE a=55; } {55 55 55 55} do_execsql_test autoanalyze1-201 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} do_execsql_test autoanalyze1-210 { SELECT * FROM t1 WHERE a IN (55,199,299); } {55 55 55 55} do_execsql_test autoanalyze1-211 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} do_execsql_test autoanalyze1-220 { SELECT * FROM t1 WHERE (b,c)=(45,45); } {45 45 45 45} do_execsql_test autoanalyze1-221 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} # Any use of the non-unique t1d index triggers the use of stats. @@ -85,7 +84,7 @@ do_execsql_test autoanalyze1-300 { SELECT * FROM t1 WHERE d=45; } {45 45 45 45} do_execsql_test autoanalyze1-301 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} sqlite3 db test.db @@ -93,7 +92,7 @@ do_execsql_test autoanalyze1-310 { SELECT * FROM t1 WHERE d=45 AND a=45; } {45 45 45 45} do_execsql_test autoanalyze1-311 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} ;# The ROWID lookup short-circuits the d=45 constraint sqlite3 db test.db @@ -101,7 +100,7 @@ do_execsql_test autoanalyze1-320 { SELECT * FROM t1 WHERE d=45 AND a IN (45,46); } {45 45 45 45} do_execsql_test autoanalyze1-321 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} # Any use of prefix of a unique index triggers the use of stats @@ -111,14 +110,14 @@ do_execsql_test autoanalyze1-400 { SELECT * FROM t1 WHERE b=45; } {45 45 45 45} do_execsql_test autoanalyze1-401 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} # The TF_StatsUsed flag is reset when the database is reopened # sqlite3 db test.db do_execsql_test autoanalyze1-500 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} finish_test From f196972c73caf56506e570818d16d024963d5b5f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 23:52:00 +0000 Subject: [PATCH 173/292] Fix the #endif location for an #ifndef SQLITE_UNTESTABLE macro in the command-line shell. FossilOrigin-Name: 8cc9d74c176a78aeebfbb39198c21b5dd547ff52 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dc0eaa5c63..f6dcc6d61d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sthat\swas\smade\sto\sfail\sby\sthe\sLIKE\soptimization\senhancement\nin\scheck-in\s[158290c0ab]\sbut\swhich\swent\sunnoticed\sbecause\stest\sbuilds\swere\nrunning\swith\sICU\senabled\sand\sICU\sdisables\sthe\sLIKE\soptimization. -D 2017-02-17T02:04:31.068 +C Fix\sthe\s#endif\slocation\sfor\san\s#ifndef\sSQLITE_UNTESTABLE\smacro\sin\sthe\ncommand-line\sshell. +D 2017-02-17T23:52:00.611 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -395,7 +395,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 +F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a03be1dc42737ba0712d33f639ea26dc243b20e -R b29803fdd4fd514ba7baa53a58b26311 +P 218b2bbb0de07288889f6762d4461ea8acd78969 +R ac73be2e300a373fc7f110abc87089ba U drh -Z af65459984ca8c135fabce3c62812860 +Z 57022d1bdb1e12a8cd90064785570f14 diff --git a/manifest.uuid b/manifest.uuid index d642d67abe..ce8096c87e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -218b2bbb0de07288889f6762d4461ea8acd78969 \ No newline at end of file +8cc9d74c176a78aeebfbb39198c21b5dd547ff52 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index fbbbea9d79..f1c7f3c157 100644 --- a/src/shell.c +++ b/src/shell.c @@ -5192,6 +5192,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } }else +#endif /* !defined(SQLITE_UNTESTABLE) */ if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); @@ -5228,7 +5229,6 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else -#endif /* !defined(SQLITE_UNTESTABLE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ From 182e84c116eb9b6f3e9dc3ee1a694e302cf6c08d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 02:19:02 +0000 Subject: [PATCH 174/292] In the analyze_as_needed pragma, avoid running unnecessary OP_LoadAnalysis and OP_Expire opcodes. Make the analyze_as_needed pragma responsive to the schema name. FossilOrigin-Name: 882599a4a7ea92c9e7752e0745475508e58a11c3 --- manifest | 18 +++++----- manifest.uuid | 2 +- src/analyze.c | 90 ++++++++++++++++++++++++++----------------------- src/parse.y | 4 +-- src/pragma.c | 2 +- src/sqliteInt.h | 3 +- 6 files changed, 63 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index 42a11a725c..d311cdbbf3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sanalyze_as_needed\spragma\snow\sresponds\sto\stable\ssize\sgrowth\sand\swill\nautomatically\srerun\sthe\sanalysis\safter\seach\s10x\ssize\sincrease. -D 2017-02-17T19:24:06.618 +C In\sthe\sanalyze_as_needed\spragma,\savoid\srunning\sunnecessary\sOP_LoadAnalysis\s\nand\sOP_Expire\sopcodes.\s\sMake\sthe\sanalyze_as_needed\spragma\sresponsive\sto\sthe\nschema\sname. +D 2017-02-18T02:19:02.183 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c acef9c48eac30438e82f39eb8df6682c6e33942e +F src/analyze.c eb50045b8f2e0d8a0a36a2158a65afe098d9a3bb F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -383,11 +383,11 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y d5695ae4887a52352b1b8a3e3623477c7e79abeb +F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c e8b2ea66dfb1b90c53b920c08914ac26b109053a +F src/pragma.c cf0f101d2986d258699cbbd249392858ec9a6b7e F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 60c38b84a796cc9d0a9a794cca254b09898b9cc5 +F src/sqliteInt.h 70abfa92e02688f1d189bf63fd25d309ca8d8d1e F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e93db2373127d31d33ec46ef918fa9386bb664a6 -R 484e61dbf95be1098adb3acfa069e723 +P bfbdd07409688fac4ccddbab3639745f6152e23d +R 1c1b7451b307830f90322e6d51007af7 U drh -Z 81466028927592a195c5763d3460fcde +Z 2d7c806c83a06bb426974433c00a5916 diff --git a/manifest.uuid b/manifest.uuid index 8be7eeb5b9..9825130b6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bfbdd07409688fac4ccddbab3639745f6152e23d \ No newline at end of file +882599a4a7ea92c9e7752e0745475508e58a11c3 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 082713fd23..c30027aec9 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -971,6 +971,7 @@ static void analyzeOneTable( int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ + int regWorkDone = iMem++; /* Set to 1 if any work is done */ int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ @@ -1018,7 +1019,7 @@ static void analyzeOneTable( sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); if( szOld>0 ){ addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); - } + } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1276,21 +1277,11 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } + sqlite3VdbeAddOp2(v, OP_Integer, 1, regWorkDone); + VdbeComment((v, "work was done")); sqlite3VdbeJumpHere(v, addrSizeCk); } - -/* -** Generate code that will cause the most recent index analysis to -** be loaded into internal hash tables where is can be used. -*/ -static void loadAnalysis(Parse *pParse, int iDb){ - Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); - } -} - /* ** Return true if table pTab might need to being reanalyzed. Return ** false if we know that pTab should not be reanalyzed. @@ -1322,54 +1313,70 @@ static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ } /* -** Generate code that will do an analysis of an entire database. +** Generate code that will do an analysis of an entire database, or +** all databases in the connection if iDbReq is negative. ** -** Return a count of the number of tables actually analyzed. Return 0 -** if nothing was analyzed. +** If onlyIfNeeded is true, then only run the analysis if SQLite thinks +** it is actually needed. */ -static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ +void sqlite3AnalyzeDatabase( + Parse *pParse, /* The parsing context */ + int iDbReq, /* Which schema to analyze. -1 for all (except TEMP) */ + int onlyIfNeeded /* Only do the analysis if needed, when true */ +){ sqlite3 *db = pParse->db; Schema *pSchema; HashElem *k; - int iStatCur; - int iMem; - int iTab; + int iStatCur = 0; + int iMem = 0; + int iTab = 0; int iDb; /* Database currently being analyzed */ int iDbFirst, iDbLast; /* Range of databases to be analyzed */ - int cnt; /* Number of tables analyzed in a single database */ - int allCnt = 0; /* Number of tables analyzed across all databases */ + int bStatTabs = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + int nHit = 0; + unsigned char aHit[SQLITE_MAX_ATTACHED+2]; + if( v==0 ) return; if( iDbReq>=0 ){ iDbFirst = iDbLast = iDbReq; }else{ iDbFirst = 0; iDbLast = db->nDb-1; } - for(iDb=iDbFirst; iDb<=iDbLast; iDb++){ + for(iDb=iDbFirst; iDb<=iDbLast; iDb++, bStatTabs=0){ if( iDb==1 ) continue; /* Do not analyze the TEMP database */ pSchema = db->aDb[iDb].pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - cnt = 0; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); LogEst szThreshold = 0; if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ - if( cnt==0 ){ - sqlite3BeginWriteOperation(pParse, 0, iDb); + if( iMem==0 ){ iStatCur = pParse->nTab; pParse->nTab += 3; - openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; iTab = pParse->nTab; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem); + } + if( !bStatTabs ){ + aHit[nHit++] = iDb; + sqlite3BeginWriteOperation(pParse, 0, iDb); + openStatTable(pParse, iDb, iStatCur, 0, 0); + bStatTabs = 1; } analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); - cnt++; - allCnt++; } } - if( cnt ) loadAnalysis(pParse, iDb); } - return allCnt; + if( iMem ){ + int addrTop = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); VdbeCoverage(v); + for(iDb=0; iDbdb) ); @@ -1392,9 +1400,13 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1, + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1, pParse->nTab, 0); - loadAnalysis(pParse, iDb); + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + sqlite3VdbeAddOp0(v, OP_Expire); + } } /* @@ -1414,15 +1426,13 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** are believed to be in need of analysis. The conditional analysis ** might well be a no-op. */ -void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ +void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ sqlite3 *db = pParse->db; int iDb; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; - Vdbe *v; - int needExpire = 1; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1434,12 +1444,12 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - needExpire = analyzeDatabase(pParse, -1, ifNeeded) || ifNeeded==0; + sqlite3AnalyzeDatabase(pParse, -1, 0); }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - analyzeDatabase(pParse, iDb, 0); + sqlite3AnalyzeDatabase(pParse, iDb, 0); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1467,10 +1477,6 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ } } } - if( needExpire ){ - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); - } } /* diff --git a/src/parse.y b/src/parse.y index ff03c128ac..9cada2a1be 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1512,8 +1512,8 @@ cmd ::= REINDEX nm(X) dbnm(Y). {sqlite3Reindex(pParse, &X, &Y);} /////////////////////////////////// ANALYZE /////////////////////////////////// %ifndef SQLITE_OMIT_ANALYZE -cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0, 0);} -cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y, 0);} +cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0);} +cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} %endif //////////////////////// ALTER TABLE table ... //////////////////////////////// diff --git a/src/pragma.c b/src/pragma.c index 789751958c..3bcf62f69f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1832,7 +1832,7 @@ void sqlite3Pragma( ** PRAGMA analyze_as_needed */ case PragTyp_ANALYZE_AS_NEEDED: { - sqlite3Analyze(pParse, 0, 0, 1); + sqlite3AnalyzeDatabase(pParse, zDb ? iDb : -1, 1); break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d9cad11bcf..218c2c6af6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4027,7 +4027,8 @@ void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); -void sqlite3Analyze(Parse*, Token*, Token*, int); +void sqlite3Analyze(Parse*, Token*, Token*); +void sqlite3AnalyzeDatabase(Parse*,int,int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); From dfe11bae9944fa05066314dd9063ff834fd60dae Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 02:42:54 +0000 Subject: [PATCH 175/292] Fix errors in the table resize detection. FossilOrigin-Name: 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 8 +++++--- src/btree.c | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d311cdbbf3..5af18e20b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sanalyze_as_needed\spragma,\savoid\srunning\sunnecessary\sOP_LoadAnalysis\s\nand\sOP_Expire\sopcodes.\s\sMake\sthe\sanalyze_as_needed\spragma\sresponsive\sto\sthe\nschema\sname. -D 2017-02-18T02:19:02.183 +C Fix\serrors\sin\sthe\stable\sresize\sdetection. +D 2017-02-18T02:42:54.892 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,13 +331,13 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c eb50045b8f2e0d8a0a36a2158a65afe098d9a3bb +F src/analyze.c e01e5362ba2ac8430ab7833022afb6081ade8315 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c e7d724b02365d88920171caf0c919aa96d9b19e3 +F src/btree.c a4ab1fb5cdeea88c4f76216e41cfecfa505c8c43 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bfbdd07409688fac4ccddbab3639745f6152e23d -R 1c1b7451b307830f90322e6d51007af7 +P 882599a4a7ea92c9e7752e0745475508e58a11c3 +R cf9f80fd407d6e9e7cfac9c2957017b5 U drh -Z 2d7c806c83a06bb426974433c00a5916 +Z 225df65ba1e601aba9cbc975df68b3a8 diff --git a/manifest.uuid b/manifest.uuid index 9825130b6a..07d49087a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -882599a4a7ea92c9e7752e0745475508e58a11c3 \ No newline at end of file +4229caec0b60a1617b9d5ff94b47271cbd7be1e0 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c30027aec9..118cdb9383 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1019,6 +1019,7 @@ static void analyzeOneTable( sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); if( szOld>0 ){ addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); + VdbeCoverage(v); } sqlite3VdbeLoadString(v, regTabname, pTab->zName); @@ -1296,9 +1297,10 @@ static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ Index *pIdx; if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - /* If TF_StatsUsed is true, then we might need to reanalyze. But - ** only reanalyze if the table size has grown by a factor of 10 or more */ - *pThreshold = pTab->nRowLogEst + 33; assert( sqlite3LogEst(10)==33 ); + /* If TF_StatsUsed is true, then we might need to reanalyze. + ** TUNING: Only reanalyze if the table size has grown by a factor + ** of 25 or more. */ + *pThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); /* Except, if any of the indexes of the table do not have valid ** sqlite_stat1 entries, then set the size threshold to zero to diff --git a/src/btree.c b/src/btree.c index 8aed08cb36..4d31661f10 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5331,8 +5331,8 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); if( pCur->eState!=CURSOR_VALID ) return -1; - if( pCur->apPage[pCur->iPage-1]->leaf==0 ) return -1; - for(n=1, i=0; iiPage; i++){ + if( pCur->apPage[pCur->iPage]->leaf==0 ) return -1; + for(n=1, i=0; i<=pCur->iPage; i++){ n *= pCur->apPage[i]->nCell; } return n; From eeea412a9d2b26f9f82157899520e3ad2e2ee34f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 13:47:11 +0000 Subject: [PATCH 176/292] Add the SQLITE_BUG_COMPATIBLE_20160819 compile-time option to omit the error message when an unrecognized argument is provided to the VACUUM command. FossilOrigin-Name: 491814272dce7e937b4734fcbc2ad69e12377b56 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vacuum.c | 21 +++++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f6dcc6d61d..44763338e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s#endif\slocation\sfor\san\s#ifndef\sSQLITE_UNTESTABLE\smacro\sin\sthe\ncommand-line\sshell. -D 2017-02-17T23:52:00.611 +C Add\sthe\sSQLITE_BUG_COMPATIBLE_20160819\scompile-time\soption\sto\somit\sthe\serror\nmessage\swhen\san\sunrecognized\sargument\sis\sprovided\sto\sthe\sVACUUM\scommand. +D 2017-02-18T13:47:11.181 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -460,7 +460,7 @@ F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 -F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 +F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 218b2bbb0de07288889f6762d4461ea8acd78969 -R ac73be2e300a373fc7f110abc87089ba +P 8cc9d74c176a78aeebfbb39198c21b5dd547ff52 +R 57839b570efd132c834489fe34d93e5e U drh -Z 57022d1bdb1e12a8cd90064785570f14 +Z 38cfd439ab0cfeae01cbf1d291727475 diff --git a/manifest.uuid b/manifest.uuid index ce8096c87e..557d9a7e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cc9d74c176a78aeebfbb39198c21b5dd547ff52 \ No newline at end of file +491814272dce7e937b4734fcbc2ad69e12377b56 \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 25b1258510..9e471b8d9c 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -98,8 +98,25 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ */ void sqlite3Vacuum(Parse *pParse, Token *pNm){ Vdbe *v = sqlite3GetVdbe(pParse); - int iDb = pNm ? sqlite3TwoPartName(pParse, pNm, pNm, &pNm) : 0; - if( v && (iDb>=2 || iDb==0) ){ + int iDb = 0; + if( v==0 ) return; + if( pNm ){ +#ifndef SQLITE_BUG_COMPATIBLE_20160819 + /* Default behavior: Report an error if the argument to VACUUM is + ** not recognized */ + iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); + if( iDb<0 ) return; +#else + /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments + ** to VACUUM are silently ignored. This is a back-out of a bug fix that + ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** The buggy behavior is required for binary compatibility with some + ** legacy applications. */ + iDb = sqlite3FindDb(pParse->db, pNm); + if( iDb<0 ) iDb = 0; +#endif + } + if( iDb!=1 ){ sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); sqlite3VdbeUsesBtree(v, iDb); } From 4a54bb579480352dce25d2982b2f266c58255248 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 15:58:52 +0000 Subject: [PATCH 177/292] Add the OP_SqlExec opcode and use it to implement "PRAGMA analyze_as_needed", invoking ANALYZE subcommands as necessary. This simplifies the implementation. FossilOrigin-Name: d386015f5e7ecdd951d70db56b7bbd858be7ad90 --- manifest | 18 +++--- manifest.uuid | 2 +- src/analyze.c | 157 +++++++++++++----------------------------------- src/pragma.c | 36 ++++++++++- src/sqliteInt.h | 1 - src/vdbe.c | 10 +++ 6 files changed, 96 insertions(+), 128 deletions(-) diff --git a/manifest b/manifest index 5af18e20b2..984bdbb4fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\serrors\sin\sthe\stable\sresize\sdetection. -D 2017-02-18T02:42:54.892 +C Add\sthe\sOP_SqlExec\sopcode\sand\suse\sit\sto\simplement\s"PRAGMA\sanalyze_as_needed",\ninvoking\sANALYZE\ssubcommands\sas\snecessary.\s\sThis\ssimplifies\sthe\simplementation. +D 2017-02-18T15:58:52.451 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c e01e5362ba2ac8430ab7833022afb6081ade8315 +F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -387,7 +387,7 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c cf0f101d2986d258699cbbd249392858ec9a6b7e +F src/pragma.c 1ed159f6fed5af7f5b095f55bdefcb0848956397 F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 70abfa92e02688f1d189bf63fd25d309ca8d8d1e +F src/sqliteInt.h 87857c2b8d8825e22970ce3f466b8618ee55b44e F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c ac8538b96af5c63d6f18484ffd981d4f44db0073 +F src/vdbe.c 02f9db522c73ba4e1743585e9fd6f8ac6c71515b F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 882599a4a7ea92c9e7752e0745475508e58a11c3 -R cf9f80fd407d6e9e7cfac9c2957017b5 +P 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 +R 03be76497e56750b0101f816e3587990 U drh -Z 225df65ba1e601aba9cbc975df68b3a8 +Z 8d178351b984357dd4d18d50fe476798 diff --git a/manifest.uuid b/manifest.uuid index 07d49087a1..323aca9202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4229caec0b60a1617b9d5ff94b47271cbd7be1e0 \ No newline at end of file +d386015f5e7ecdd951d70db56b7bbd858be7ad90 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 118cdb9383..3ecf469cd3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -958,12 +958,10 @@ static void analyzeOneTable( Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ int iMem, /* Available memory locations begin here */ - int iTab, /* Next available cursor */ - LogEst szOld /* Run the analysis if table row count is larger than this */ + int iTab /* Next available cursor */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ - int addrSizeCk = 0; /* Address of the IfSmaller */ int iIdxCur; /* Cursor open on index being analyzed */ int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ @@ -971,7 +969,6 @@ static void analyzeOneTable( int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ - int regWorkDone = iMem++; /* Set to 1 if any work is done */ int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ @@ -1017,10 +1014,6 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - if( szOld>0 ){ - addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); - VdbeCoverage(v); - } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1278,107 +1271,43 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regWorkDone); - VdbeComment((v, "work was done")); - sqlite3VdbeJumpHere(v, addrSizeCk); } -/* -** Return true if table pTab might need to being reanalyzed. Return -** false if we know that pTab should not be reanalyzed. -** -** If returning true, also set *pThreshold to a size threshold that -** will determine at run-time whether or not the reanalysis occurs. -** The reanalysis will only occur if the size of the table is greater -** than the threshold. Not that the threshold is a logarithmic LogEst -** value. -*/ -static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ - Index *pIdx; - if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - - /* If TF_StatsUsed is true, then we might need to reanalyze. - ** TUNING: Only reanalyze if the table size has grown by a factor - ** of 25 or more. */ - *pThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); - - /* Except, if any of the indexes of the table do not have valid - ** sqlite_stat1 entries, then set the size threshold to zero to - ** ensure the analysis will always occur. */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 ){ - *pThreshold = 0; - break; - } - } - return 1; -} /* -** Generate code that will do an analysis of an entire database, or -** all databases in the connection if iDbReq is negative. -** -** If onlyIfNeeded is true, then only run the analysis if SQLite thinks -** it is actually needed. +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. */ -void sqlite3AnalyzeDatabase( - Parse *pParse, /* The parsing context */ - int iDbReq, /* Which schema to analyze. -1 for all (except TEMP) */ - int onlyIfNeeded /* Only do the analysis if needed, when true */ -){ - sqlite3 *db = pParse->db; - Schema *pSchema; - HashElem *k; - int iStatCur = 0; - int iMem = 0; - int iTab = 0; - int iDb; /* Database currently being analyzed */ - int iDbFirst, iDbLast; /* Range of databases to be analyzed */ - int bStatTabs = 0; +static void loadAnalysis(Parse *pParse, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); - int nHit = 0; - unsigned char aHit[SQLITE_MAX_ATTACHED+2]; + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + } +} - if( v==0 ) return; - if( iDbReq>=0 ){ - iDbFirst = iDbLast = iDbReq; - }else{ - iDbFirst = 0; - iDbLast = db->nDb-1; - } - for(iDb=iDbFirst; iDb<=iDbLast; iDb++, bStatTabs=0){ - if( iDb==1 ) continue; /* Do not analyze the TEMP database */ - pSchema = db->aDb[iDb].pSchema; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ - Table *pTab = (Table*)sqliteHashData(k); - LogEst szThreshold = 0; - if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ - if( iMem==0 ){ - iStatCur = pParse->nTab; - pParse->nTab += 3; - iMem = pParse->nMem+1; - iTab = pParse->nTab; - sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem); - } - if( !bStatTabs ){ - aHit[nHit++] = iDb; - sqlite3BeginWriteOperation(pParse, 0, iDb); - openStatTable(pParse, iDb, iStatCur, 0, 0); - bStatTabs = 1; - } - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); - } - } - } - if( iMem ){ - int addrTop = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); VdbeCoverage(v); - for(iDb=0; iDbdb; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; + int iTab; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); } + loadAnalysis(pParse, iDb); } /* @@ -1389,7 +1318,6 @@ void sqlite3AnalyzeDatabase( static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ int iDb; int iStatCur; - Vdbe *v; assert( pTab!=0 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); @@ -1402,13 +1330,8 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1, - pParse->nTab, 0); - v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); - sqlite3VdbeAddOp0(v, OP_Expire); - } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + loadAnalysis(pParse, iDb); } /* @@ -1422,19 +1345,16 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** Form 1 causes all indices in all attached databases to be analyzed. ** Form 2 analyzes all indices the single database named. ** Form 3 analyzes all indices associated with the named table. -** -** If pName1 and pName2 are both NULL and if the ifNeeded flag is true, -** this routine computes an conditional ANALYZE on only those tables -** are believed to be in need of analysis. The conditional analysis -** might well be a no-op. */ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ sqlite3 *db = pParse->db; int iDb; + int i; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1446,12 +1366,15 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - sqlite3AnalyzeDatabase(pParse, -1, 0); + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); + } }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - sqlite3AnalyzeDatabase(pParse, iDb, 0); + analyzeDatabase(pParse, iDb); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1479,6 +1402,8 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* diff --git a/src/pragma.c b/src/pragma.c index 3bcf62f69f..49d711ded7 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1832,7 +1832,41 @@ void sqlite3Pragma( ** PRAGMA analyze_as_needed */ case PragTyp_ANALYZE_AS_NEEDED: { - sqlite3AnalyzeDatabase(pParse, zDb ? iDb : -1, 1); + int iDbLast; + int iTabCur; + HashElem *k; + Schema *pSchema; + Table *pTab; + Index *pIdx; + LogEst szThreshold; + char *zSubSql; + + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2, szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 218c2c6af6..19c7481223 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4028,7 +4028,6 @@ void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); void sqlite3Analyze(Parse*, Token*, Token*); -void sqlite3AnalyzeDatabase(Parse*,int,int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); diff --git a/src/vdbe.c b/src/vdbe.c index b595c4fb01..b8050e8d50 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5518,6 +5518,16 @@ case OP_CreateTable: { /* out2 */ break; } +/* Opcode: SqlExec * * * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +*/ +case OP_SqlExec: { + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + if( rc ) goto abort_due_to_error; + break; +} + /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 From 99d5b4cde6c64a514d9531494bb6875bf5b2a46c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 22:52:40 +0000 Subject: [PATCH 178/292] Updated comments. No code changes. FossilOrigin-Name: e842ad391e62df273a5b1ed569d42ea46d03a99b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 41 ++++++++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 984bdbb4fc..420fc4058d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sOP_SqlExec\sopcode\sand\suse\sit\sto\simplement\s"PRAGMA\sanalyze_as_needed",\ninvoking\sANALYZE\ssubcommands\sas\snecessary.\s\sThis\ssimplifies\sthe\simplementation. -D 2017-02-18T15:58:52.451 +C Updated\scomments.\s\sNo\scode\schanges. +D 2017-02-18T22:52:40.654 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -387,7 +387,7 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 1ed159f6fed5af7f5b095f55bdefcb0848956397 +F src/pragma.c 8a35509e8f60746fbdcde5312ddfd177c46c7e9c F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 -R 03be76497e56750b0101f816e3587990 +P d386015f5e7ecdd951d70db56b7bbd858be7ad90 +R bb5056af5425096e2e052e924ae83b2e U drh -Z 8d178351b984357dd4d18d50fe476798 +Z a573182d78be17d037798d5840f110fc diff --git a/manifest.uuid b/manifest.uuid index 323aca9202..d08504dc8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d386015f5e7ecdd951d70db56b7bbd858be7ad90 \ No newline at end of file +e842ad391e62df273a5b1ed569d42ea46d03a99b \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 49d711ded7..3aa07e8d18 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1830,16 +1830,33 @@ void sqlite3Pragma( /* ** PRAGMA analyze_as_needed + ** PRAGMA schema.analyze_as_needed + ** + ** This pragma runs ANALYZE on any tables which would have benefitted + ** from having recent statistics at some point since the start of the + ** current connection. Only tables in "schema" are analyzed in the + ** second form. In the first form, all tables except TEMP tables are + ** checked. + ** + ** A table is analyzed only if both of the following are true: + ** + ** (1) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (2) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. */ case PragTyp_ANALYZE_AS_NEEDED: { - int iDbLast; - int iTabCur; - HashElem *k; - Schema *pSchema; - Table *pTab; - Index *pIdx; - LogEst szThreshold; - char *zSubSql; + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ @@ -1848,11 +1865,17 @@ void sqlite3Pragma( pSchema = db->aDb[iDb].pSchema; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !pIdx->hasStat1 ){ - szThreshold = 0; + szThreshold = 0; /* Always analyze if any index lacks statistics */ break; } } From 7e84b377a911ae9fb882dcf9eba23f90b59a849c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Feb 2017 14:30:17 +0000 Subject: [PATCH 179/292] Small grammar simplification. FossilOrigin-Name: 0d8a868acd74fb1d076f23fda58b841bb7e6900b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/parse.y | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 44763338e4..ab58b869fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_BUG_COMPATIBLE_20160819\scompile-time\soption\sto\somit\sthe\serror\nmessage\swhen\san\sunrecognized\sargument\sis\sprovided\sto\sthe\sVACUUM\scommand. -D 2017-02-18T13:47:11.181 +C Small\sgrammar\ssimplification. +D 2017-02-20T14:30:17.816 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -383,7 +383,7 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c +F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8cc9d74c176a78aeebfbb39198c21b5dd547ff52 -R 57839b570efd132c834489fe34d93e5e +P 491814272dce7e937b4734fcbc2ad69e12377b56 +R b4cf79fa4c0fd30460236bc8c090e993 U drh -Z 38cfd439ab0cfeae01cbf1d291727475 +Z 32b02aa6638f1e40c7df4614e1f90d4a diff --git a/manifest.uuid b/manifest.uuid index 557d9a7e8b..c411ab59d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -491814272dce7e937b4734fcbc2ad69e12377b56 \ No newline at end of file +0d8a868acd74fb1d076f23fda58b841bb7e6900b \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 9cada2a1be..0949e97d16 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1000,7 +1000,7 @@ expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} expr(A) ::= expr(A) CONCAT(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} %type likeop {Token} -likeop(A) ::= LIKE_KW|MATCH(X). {A=X;/*A-overwrites-X*/} +likeop(A) ::= LIKE_KW|MATCH(A). likeop(A) ::= NOT LIKE_KW|MATCH(X). {A=X; A.n|=0x80000000; /*A-overwrite-X*/} expr(A) ::= expr(A) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; From 5e6710ab6540e45cfb79d85609f28c496d5b6f25 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 20 Feb 2017 19:13:37 +0000 Subject: [PATCH 180/292] Avoid unsigned integer overflows for SQLITE_WIN32_HEAP_INIT_SIZE when the Win32 heap subsystem is used with very large values of SQLITE_DEFAULT_CACHE_SIZE and/or SQLITE_DEFAULT_PAGE_SIZE. FossilOrigin-Name: 96b6a98e5e4cb0ddbfcd78b05bfbfcd8976e9f32 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 52 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index ab58b869fe..34aa979a22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sgrammar\ssimplification. -D 2017-02-20T14:30:17.816 +C Avoid\sunsigned\sinteger\soverflows\sfor\sSQLITE_WIN32_HEAP_INIT_SIZE\swhen\sthe\sWin32\sheap\ssubsystem\sis\sused\swith\svery\slarge\svalues\sof\sSQLITE_DEFAULT_CACHE_SIZE\sand/or\sSQLITE_DEFAULT_PAGE_SIZE. +D 2017-02-20T19:13:37.359 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -379,7 +379,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 -F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 +F src/os_win.c c97c79fe19dfb0a14c89b78280beabd9ac28acb1 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 491814272dce7e937b4734fcbc2ad69e12377b56 -R b4cf79fa4c0fd30460236bc8c090e993 -U drh -Z 32b02aa6638f1e40c7df4614e1f90d4a +P 0d8a868acd74fb1d076f23fda58b841bb7e6900b +R 42e490f947871a7f92e95baf4bc40d36 +U mistachkin +Z 144582bee7eff0d5ee39c945bf2670f2 diff --git a/manifest.uuid b/manifest.uuid index c411ab59d6..60aeb2563f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d8a868acd74fb1d076f23fda58b841bb7e6900b \ No newline at end of file +96b6a98e5e4cb0ddbfcd78b05bfbfcd8976e9f32 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2cb5f7b0c8..8d75d8bec5 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -352,7 +352,34 @@ struct winVfsAppData { ****************************************************************************** */ #ifndef SQLITE_WIN32_HEAP_CREATE -# define SQLITE_WIN32_HEAP_CREATE (TRUE) +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) #endif /* @@ -361,25 +388,36 @@ struct winVfsAppData { */ #ifndef SQLITE_WIN32_CACHE_SIZE # if SQLITE_DEFAULT_CACHE_SIZE>=0 -# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) # else -# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) # endif #endif +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>=SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) +#endif + /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ - (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) #endif /* * The maximum size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_MAX_SIZE -# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) #endif /* @@ -387,7 +425,7 @@ struct winVfsAppData { * zero for the default behavior. */ #ifndef SQLITE_WIN32_HEAP_FLAGS -# define SQLITE_WIN32_HEAP_FLAGS (0) +# define SQLITE_WIN32_HEAP_FLAGS (0) #endif From 4689956b9e0dfad404a0b103d7903b8cbfddb48f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 20 Feb 2017 23:32:04 +0000 Subject: [PATCH 181/292] Correct a harmless typo in the previous check-in. FossilOrigin-Name: 1589db012ef1389bf84399fccf96d143b2ac4c0f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 34aa979a22..b9e136d680 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunsigned\sinteger\soverflows\sfor\sSQLITE_WIN32_HEAP_INIT_SIZE\swhen\sthe\sWin32\sheap\ssubsystem\sis\sused\swith\svery\slarge\svalues\sof\sSQLITE_DEFAULT_CACHE_SIZE\sand/or\sSQLITE_DEFAULT_PAGE_SIZE. -D 2017-02-20T19:13:37.359 +C Correct\sa\sharmless\stypo\sin\sthe\sprevious\scheck-in. +D 2017-02-20T23:32:04.979 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -379,7 +379,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 -F src/os_win.c c97c79fe19dfb0a14c89b78280beabd9ac28acb1 +F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d8a868acd74fb1d076f23fda58b841bb7e6900b -R 42e490f947871a7f92e95baf4bc40d36 +P 96b6a98e5e4cb0ddbfcd78b05bfbfcd8976e9f32 +R a6d884232566cde130b9b77412776230 U mistachkin -Z 144582bee7eff0d5ee39c945bf2670f2 +Z c97450fc686b8c32bf5204625d8622fc diff --git a/manifest.uuid b/manifest.uuid index 60aeb2563f..a4b5b98287 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96b6a98e5e4cb0ddbfcd78b05bfbfcd8976e9f32 \ No newline at end of file +1589db012ef1389bf84399fccf96d143b2ac4c0f \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8d75d8bec5..a87d7d0925 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -399,7 +399,7 @@ struct winVfsAppData { * initial size of the Win32-specific heap to exceed the maximum amount * of memory that can be specified in the call to HeapCreate. */ -#if SQLITE_WIN32_CACHE_SIZE>=SQLITE_WIN32_MAX_CACHE_SIZE +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE # undef SQLITE_WIN32_CACHE_SIZE # define SQLITE_WIN32_CACHE_SIZE (2000) #endif From e3247822549c6ef80b33baaffb0f4698ef9861bc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Feb 2017 15:27:22 +0000 Subject: [PATCH 182/292] Very small enhancement to dispatch speed for SQL functions. FossilOrigin-Name: 3c3228ed16ed8a72630bd56bb9192ee3c7f82093 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index b9e136d680..ec707ac5ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sa\sharmless\stypo\sin\sthe\sprevious\scheck-in. -D 2017-02-20T23:32:04.979 +C Very\ssmall\senhancement\sto\sdispatch\sspeed\sfor\sSQL\sfunctions. +D 2017-02-21T15:27:22.044 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b +F src/vdbe.c 16542e327eb38763490473dc0e9e9396924f30b2 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1556,7 +1556,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96b6a98e5e4cb0ddbfcd78b05bfbfcd8976e9f32 -R a6d884232566cde130b9b77412776230 -U mistachkin -Z c97450fc686b8c32bf5204625d8622fc +P 1589db012ef1389bf84399fccf96d143b2ac4c0f +R e3362d21e61dd042dd5284f17843c5d9 +U drh +Z 12602c247389d019265df8c626a67cf1 diff --git a/manifest.uuid b/manifest.uuid index a4b5b98287..451bfd624c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1589db012ef1389bf84399fccf96d143b2ac4c0f \ No newline at end of file +3c3228ed16ed8a72630bd56bb9192ee3c7f82093 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7f286b2c6d..97aa519df1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1669,21 +1669,21 @@ case OP_Function: { for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - memAboutToChange(p, pCtx->pOut); + memAboutToChange(p, pOut); #ifdef SQLITE_DEBUG for(i=0; iargc; i++){ assert( memIsValid(pCtx->argv[i]) ); REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); } #endif - MemSetTypeFlag(pCtx->pOut, MEM_Null); + MemSetTypeFlag(pOut, MEM_Null); pCtx->fErrorOrAux = 0; (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ /* If the function returned an error, throw an exception */ if( pCtx->fErrorOrAux ){ if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); rc = pCtx->isError; } sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); @@ -1692,12 +1692,12 @@ case OP_Function: { /* Copy the result of the function into register P3 */ if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); - if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; + sqlite3VdbeChangeEncoding(pOut, encoding); + if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; } - REGISTER_TRACE(pOp->p3, pCtx->pOut); - UPDATE_MAX_BLOBSIZE(pCtx->pOut); + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); break; } From f0e808c60b66cae45cbcf3c0fffb1e08776e879e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Feb 2017 17:52:58 +0000 Subject: [PATCH 183/292] Fix an FTS5 problem that could cause a crash when certain queries were interrupted using sqlite3_interrupt(). FossilOrigin-Name: e400909f313c317b7b67be6eb867ed61df7383dc --- ext/fts5/fts5_expr.c | 15 +++++- ext/fts5/test/fts5faultD.test | 87 +++++++++++++++++++++++++++++++++++ manifest | 15 +++--- manifest.uuid | 2 +- 4 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 ext/fts5/test/fts5faultD.test diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 209748bbf5..e18691ad0b 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1110,7 +1110,10 @@ static int fts5ExprNodeNext_OR( || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } } } } @@ -1141,7 +1144,10 @@ static int fts5ExprNodeTest_AND( if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } } /* If the child node is now at EOF, so is the parent AND node. Otherwise, @@ -1180,6 +1186,8 @@ static int fts5ExprNodeNext_AND( int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; } return rc; } @@ -1222,6 +1230,9 @@ static int fts5ExprNodeNext_NOT( if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_NOT(pExpr, pNode); } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } return rc; } diff --git a/ext/fts5/test/fts5faultD.test b/ext/fts5/test/fts5faultD.test new file mode 100644 index 0000000000..e259cbf610 --- /dev/null +++ b/ext/fts5/test/fts5faultD.test @@ -0,0 +1,87 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultA + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + if {"%DETAIL%"=="none"} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s; + } + + do_faultsim_test 1 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('a') } + } -test { + faultsim_test_result {0 180} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 2 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 3 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 4 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('b:b OR a:a') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 5 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('c:b') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } +} + +finish_test diff --git a/manifest b/manifest index ec707ac5ad..c5dbc5dfad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\ssmall\senhancement\sto\sdispatch\sspeed\sfor\sSQL\sfunctions. -D 2017-02-21T15:27:22.044 +C Fix\san\sFTS5\sproblem\sthat\scould\scause\sa\scrash\swhen\scertain\squeries\swere\ninterrupted\susing\ssqlite3_interrupt(). +D 2017-02-21T17:52:58.643 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -102,7 +102,7 @@ F ext/fts5/fts5Int.h c629b24d2b92b99596f3b8e82289fddca06df6e1 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c 33f0e8063ac7360bcd71c0ff0dcbacdc05fffe50 +F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 @@ -160,6 +160,7 @@ F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca +F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d @@ -1556,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1589db012ef1389bf84399fccf96d143b2ac4c0f -R e3362d21e61dd042dd5284f17843c5d9 -U drh -Z 12602c247389d019265df8c626a67cf1 +P 3c3228ed16ed8a72630bd56bb9192ee3c7f82093 +R c2d6d6c70c7c0d8b681ef9ca8535c3a3 +U dan +Z 2a4ed498ce9ed73234b51fb6f9a899a9 diff --git a/manifest.uuid b/manifest.uuid index 451bfd624c..94de19d8f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c3228ed16ed8a72630bd56bb9192ee3c7f82093 \ No newline at end of file +e400909f313c317b7b67be6eb867ed61df7383dc \ No newline at end of file From 1325adf5e9d6745c9e00a85e294025ceda0691e9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Feb 2017 21:24:05 +0000 Subject: [PATCH 184/292] In sqlite3VdbeHalt(), return as soon as possible if Vdbe.magic!=VDBE_MAGIC_RUN. This makes sqlite3_reset() slightly faster in some cases. FossilOrigin-Name: 80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c5dbc5dfad..0c80aaffba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sFTS5\sproblem\sthat\scould\scause\sa\scrash\swhen\scertain\squeries\swere\ninterrupted\susing\ssqlite3_interrupt(). -D 2017-02-21T17:52:58.643 +C In\ssqlite3VdbeHalt(),\sreturn\sas\ssoon\sas\spossible\sif\nVdbe.magic!=VDBE_MAGIC_RUN.\sThis\smakes\ssqlite3_reset()\sslightly\sfaster\sin\ssome\ncases. +D 2017-02-21T21:24:05.029 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -466,7 +466,7 @@ F src/vdbe.c 16542e327eb38763490473dc0e9e9396924f30b2 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 -F src/vdbeaux.c 4122458d33318ab039c4b5da1ca4e7c9221c38e4 +F src/vdbeaux.c 2f48204a0f2875b098ee046bba9265907297b0b5 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c3228ed16ed8a72630bd56bb9192ee3c7f82093 -R c2d6d6c70c7c0d8b681ef9ca8535c3a3 +P e400909f313c317b7b67be6eb867ed61df7383dc +R 5434ea5a08220297aa4d56dd68f46447 U dan -Z 2a4ed498ce9ed73234b51fb6f9a899a9 +Z 68cf5c57f26344b5e30f88013a5c9e52 diff --git a/manifest.uuid b/manifest.uuid index 94de19d8f1..2ace79cb16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e400909f313c317b7b67be6eb867ed61df7383dc \ No newline at end of file +80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index cf062cf01d..292ff2bdd2 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2609,13 +2609,13 @@ int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ + if( p->magic!=VDBE_MAGIC_RUN ){ + return SQLITE_OK; + } if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; } closeAllCursors(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - return SQLITE_OK; - } checkActiveVdbeCnt(db); /* No commit or rollback needed if the program never started or if the From 8a284dcefe85c88be8f07997931cb8b208087a9b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2017 14:15:37 +0000 Subject: [PATCH 185/292] Enhance "PRAGMA integrity_check" so that it verifies CHECK constraints. FossilOrigin-Name: 549bae0856004ff65b505175460abd598b30fe57 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/pragma.c | 27 ++++++++++++++++++++++++++- test/check.test | 8 ++++++-- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0c80aaffba..7e7a9e3e8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\ssqlite3VdbeHalt(),\sreturn\sas\ssoon\sas\spossible\sif\nVdbe.magic!=VDBE_MAGIC_RUN.\sThis\smakes\ssqlite3_reset()\sslightly\sfaster\sin\ssome\ncases. -D 2017-02-21T21:24:05.029 +C Enhance\s"PRAGMA\sintegrity_check"\sso\sthat\sit\sverifies\sCHECK\sconstraints. +D 2017-02-22T14:15:37.561 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,7 +388,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d +F src/pragma.c aa4af3d8c4d9086cbceeafcbb8a5e75605a1207d F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -576,7 +576,7 @@ F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef -F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 +F test/check.test 92b23a91fb7be12fba7ee9ce518217e2919a21da F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1557,7 +1557,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e400909f313c317b7b67be6eb867ed61df7383dc -R 5434ea5a08220297aa4d56dd68f46447 -U dan -Z 68cf5c57f26344b5e30f88013a5c9e52 +P 80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 +R b17279d0fcfdb924feca449c6ae8dc40 +T *branch * integrity-check-improvements +T *sym-integrity-check-improvements * +T -sym-trunk * +U drh +Z 83b8ae4f812a6d4117d2cc9c8ae430b2 diff --git a/manifest.uuid b/manifest.uuid index 2ace79cb16..8a2cc1e157 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 \ No newline at end of file +549bae0856004ff65b505175460abd598b30fe57 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index b1775a4082..c7d092b30d 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1481,7 +1481,7 @@ void sqlite3Pragma( int iDataCur, iIdxCur; int r1 = -1; - if( pTab->pIndex==0 ) continue; + if( pTab->pIndex==0 && pTab->pCheck==0 ) continue; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ VdbeCoverage(v); @@ -1517,6 +1517,31 @@ void sqlite3Pragma( sqlite3VdbeJumpHere(v, jmp2); sqlite3VdbeJumpHere(v, jmp3); } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(v); + int addrCkOk = sqlite3VdbeMakeLabel(v); + ExprList *pCheck = pTab->pCheck; + char *zErr; + int k; + pParse->iSelfTab = iDataCur; + sqlite3ExprCachePush(pParse); + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addrCkOk); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeResolveLabel(v, addrCkOk); + sqlite3ExprCachePop(pParse); + } /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; diff --git a/test/check.test b/test/check.test index 43e447f70d..19f252677c 100644 --- a/test/check.test +++ b/test/check.test @@ -309,11 +309,15 @@ do_test check-4.8 { PRAGMA ignore_check_constraints=ON; UPDATE t4 SET x=0, y=1; SELECT * FROM t4; + PRAGMA integrity_check; } -} {0 1} +} {0 1 ok} +do_execsql_test check-4.8.1 { + PRAGMA ignore_check_constraints=OFF; + PRAGMA integrity_check; +} {{CHECK constraint failed in t4}} do_test check-4.9 { catchsql { - PRAGMA ignore_check_constraints=OFF; UPDATE t4 SET x=0, y=2; } } {1 {CHECK constraint failed: t4}} From 8b174f2916c16fc66c8b0248056011e2c5104e36 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2017 15:11:36 +0000 Subject: [PATCH 186/292] Fix integrity_check so that it verifies NOT NULL constraints even for tables that have no indexes. Enhance quick_check so that it verifies NOT NULL and CHECK constraints. FossilOrigin-Name: 5264844b069cdc20f456acee9f5b2b97c986120d --- manifest | 19 ++++++++---------- manifest.uuid | 2 +- src/build.c | 1 + src/pragma.c | 51 ++++++++++++++++++++++++++++++++----------------- src/sqliteInt.h | 17 +++++++++-------- 5 files changed, 52 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 7e7a9e3e8a..7776d07ac7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\s"PRAGMA\sintegrity_check"\sso\sthat\sit\sverifies\sCHECK\sconstraints. -D 2017-02-22T14:15:37.561 +C Fix\sintegrity_check\sso\sthat\sit\sverifies\sNOT\sNULL\sconstraints\seven\sfor\stables\nthat\shave\sno\sindexes.\s\sEnhance\squick_check\sso\sthat\sit\sverifies\sNOT\sNULL\sand\nCHECK\sconstraints. +D 2017-02-22T15:11:36.862 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -341,7 +341,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 -F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af +F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 @@ -388,7 +388,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c aa4af3d8c4d9086cbceeafcbb8a5e75605a1207d +F src/pragma.c ec83a3cb496d6d6d24f9de78ccbb24369dcedf48 F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -400,7 +400,7 @@ F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 46fe8e5aee3825d77fa771216ef263dc947030e7 +F src/sqliteInt.h a23e18aebdd0d851c2956a74a3a4f12ff202b472 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1557,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 -R b17279d0fcfdb924feca449c6ae8dc40 -T *branch * integrity-check-improvements -T *sym-integrity-check-improvements * -T -sym-trunk * +P 549bae0856004ff65b505175460abd598b30fe57 +R ca36fe65b7e723d2451bfac8ac59f0b7 U drh -Z 83b8ae4f812a6d4117d2cc9c8ae430b2 +Z 40e14a5b2e43e016c602364a556c9a0b diff --git a/manifest.uuid b/manifest.uuid index 8a2cc1e157..e0226c546a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -549bae0856004ff65b505175460abd598b30fe57 \ No newline at end of file +5264844b069cdc20f456acee9f5b2b97c986120d \ No newline at end of file diff --git a/src/build.c b/src/build.c index cd9c81be82..6e1e2177be 100644 --- a/src/build.c +++ b/src/build.c @@ -1115,6 +1115,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; p->aCol[p->nCol-1].notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; } /* diff --git a/src/pragma.c b/src/pragma.c index c7d092b30d..8a2b3de21e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1377,9 +1377,17 @@ void sqlite3Pragma( #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is reduced version of + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption - ** without most of the overhead of a full integrity-check. + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time wherease integrity_check is O(NlogN). */ case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; @@ -1473,7 +1481,7 @@ void sqlite3Pragma( /* Make sure all the indices are constructed correctly. */ - for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; Index *pPrior = 0; @@ -1481,7 +1489,12 @@ void sqlite3Pragma( int iDataCur, iIdxCur; int r1 = -1; - if( pTab->pIndex==0 && pTab->pCheck==0 ) continue; + if( pTab->pCheck==0 + && (pTab->tabFlags & TF_HasNotNull)==0 + && (pTab->pIndex==0 || isQuick) + ){ + continue; /* No additional checks needed for this table */ + } pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ VdbeCoverage(v); @@ -1543,7 +1556,7 @@ void sqlite3Pragma( sqlite3ExprCachePop(pParse); } /* Validate index entries for the current row */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + for(j=0, pIdx=pTab->pIndex; pIdx && !isQuick; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; int ckUniq = sqlite3VdbeMakeLabel(v); if( pPk==pIdx ) continue; @@ -1595,19 +1608,21 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeLoadString(v, 3, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); - sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + if( !isQuick ){ + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeLoadString(v, 3, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); + sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + } } #endif /* SQLITE_OMIT_BTREECOUNT */ } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9a53d33626..d9ec8c5bc9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1879,14 +1879,15 @@ struct Table { ** the TF_OOOHidden attribute would apply in this case. Such tables require ** special handling during INSERT processing. */ -#define TF_Readonly 0x01 /* Read-only system table */ -#define TF_Ephemeral 0x02 /* An ephemeral table */ -#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ -#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -/* available for reuse: 0x10 */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ +#define TF_Readonly 0x0001 /* Read-only system table */ +#define TF_Ephemeral 0x0002 /* An ephemeral table */ +#define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ +#define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ +/* available for reuse: 0x0010 */ +#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_HasNotNull 0x0100 /* Contains NOT NULL constraints */ /* From 66accfc56b51387b241ba2e27e7364ea1a01f37b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2017 18:04:42 +0000 Subject: [PATCH 187/292] Cleanup and simplification of the output row count limit control of PRAGMA integrity_check. FossilOrigin-Name: 5af7d72ed9ec758283d78ceb46627d72021c1c60 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/pragma.c | 60 +++++++++++++++++++++++---------------------------- src/vdbe.c | 6 +++--- 4 files changed, 38 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index 7776d07ac7..677bc6001d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sintegrity_check\sso\sthat\sit\sverifies\sNOT\sNULL\sconstraints\seven\sfor\stables\nthat\shave\sno\sindexes.\s\sEnhance\squick_check\sso\sthat\sit\sverifies\sNOT\sNULL\sand\nCHECK\sconstraints. -D 2017-02-22T15:11:36.862 +C Cleanup\sand\ssimplification\sof\sthe\soutput\srow\scount\slimit\scontrol\sof\nPRAGMA\sintegrity_check. +D 2017-02-22T18:04:42.473 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,7 +388,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c ec83a3cb496d6d6d24f9de78ccbb24369dcedf48 +F src/pragma.c b127edeb54c744a101b371cfa2e221fd741bcd72 F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -462,7 +462,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 16542e327eb38763490473dc0e9e9396924f30b2 +F src/vdbe.c 83f387d9e6842b1dc99f6e85bb577c5bbc4e397d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 549bae0856004ff65b505175460abd598b30fe57 -R ca36fe65b7e723d2451bfac8ac59f0b7 +P 5264844b069cdc20f456acee9f5b2b97c986120d +R 46f90a15a015e3c65c6378dae6f8ff19 U drh -Z 40e14a5b2e43e016c602364a556c9a0b +Z ac3f9bd06db21fa3e22e700d592b007d diff --git a/manifest.uuid b/manifest.uuid index e0226c546a..de6e14387f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5264844b069cdc20f456acee9f5b2b97c986120d \ No newline at end of file +5af7d72ed9ec758283d78ceb46627d72021c1c60 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 8a2b3de21e..151a7a2e82 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -295,6 +295,22 @@ static const PragmaName *pragmaLocate(const char *zName){ return lwr>upr ? 0 : &aPragmaName[mid]; } +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with the result +** held in register regResult. Decrement the result count and halt if +** the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v, int regResult){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + return addr; +} + /* ** Process a pragma statement. ** @@ -1418,7 +1434,7 @@ void sqlite3Pragma( mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } } - sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ /* Do an integrity check on each database file */ for(i=0; inDb; i++){ @@ -1433,10 +1449,6 @@ void sqlite3Pragma( if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); /* Do an integrity check of the B-Tree ** @@ -1476,7 +1488,7 @@ void sqlite3Pragma( P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); - sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + integrityCheckResultRow(v, 2); sqlite3VdbeJumpHere(v, addr); /* Make sure all the indices are constructed correctly. @@ -1496,10 +1508,6 @@ void sqlite3Pragma( continue; /* No additional checks needed for this table */ } pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); @@ -1514,21 +1522,17 @@ void sqlite3Pragma( /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; - int jmp2, jmp3; + int jmp2; if( j==pTab->iPKey ) continue; if( pTab->aCol[j].notNull==0 ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pTab->aCol[j].zName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeJumpHere(v, jmp3); } /* Verify CHECK constraints */ if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ @@ -1545,13 +1549,10 @@ void sqlite3Pragma( sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, SQLITE_JUMPIFNULL); sqlite3VdbeResolveLabel(v, addrCkFault); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", pTab->zName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addrCkOk); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + integrityCheckResultRow(v, 3); sqlite3VdbeResolveLabel(v, addrCkOk); sqlite3ExprCachePop(pParse); } @@ -1567,16 +1568,13 @@ void sqlite3Pragma( /* Verify that an index entry exists for the current table row */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + jmp4 = integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL @@ -1597,7 +1595,6 @@ void sqlite3Pragma( sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); @@ -1612,16 +1609,13 @@ void sqlite3Pragma( sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); sqlite3VdbeLoadString(v, 3, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); - sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + integrityCheckResultRow(v, 7); + sqlite3VdbeJumpHere(v, addr); } } #endif /* SQLITE_OMIT_BTREECOUNT */ @@ -1631,7 +1625,7 @@ void sqlite3Pragma( static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 4, 0}, /* 1 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, /* 3 */ }; @@ -1639,7 +1633,7 @@ void sqlite3Pragma( aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); if( aOp ){ - aOp[0].p2 = -mxErr; + aOp[0].p2 = 1-mxErr; aOp[2].p4type = P4_STATIC; aOp[2].p4.z = "ok"; } diff --git a/src/vdbe.c b/src/vdbe.c index 97aa519df1..fabe98ae57 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5611,7 +5611,7 @@ case OP_DropTrigger: { ** register P1 the text of an error message describing any problems. ** If no problems are found, store a NULL in register P1. ** -** The register P3 contains the maximum number of allowed errors. +** The register P3 contains one less than the maximum number of allowed errors. ** At most reg(P3) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. @@ -5644,14 +5644,14 @@ case OP_IntegrityCk: { assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, - (int)pnErr->u.i, &nErr); - pnErr->u.i -= nErr; + (int)pnErr->u.i+1, &nErr); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); }else if( z==0 ){ goto no_mem; }else{ + pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); } UPDATE_MAX_BLOBSIZE(pIn1); From bd1d270ec9909d48c8e5b14ee03be6600877fa9c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Feb 2017 19:27:51 +0000 Subject: [PATCH 188/292] Improve the performance of ANALYZE when SQLITE_ENABLE_STAT4 is defined. FossilOrigin-Name: 737a82444065752785c643b1d29ca097c828effb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 24 +++++++++++++++++++++--- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 061f9eb07e..5a5647725b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sPRAGMA\sintegrity_check.\nVerify\sCHECK\sconstraints.\nVerify\sNOT\sNULL\sconstraints\seven\son\stable\sthat\slack\sindexes.\nVerify\sCHECK\sand\sNOT\sNULL\sconstraints\swith\sPRAGMA\squick_check. -D 2017-02-22T18:53:13.913 +C Improve\sthe\sperformance\sof\sANALYZE\swhen\sSQLITE_ENABLE_STAT4\sis\sdefined. +D 2017-02-22T19:27:51.768 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -332,7 +332,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c ac7a5d7e3cee07d074697904e00e4a8ab7b2b4f5 +F src/analyze.c 844359e720d1116592234f2f7938432bdc5b6238 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80adc0cb4ed7bacc54b15ac8b5b205403939c8c4 5af7d72ed9ec758283d78ceb46627d72021c1c60 -R 46f90a15a015e3c65c6378dae6f8ff19 -U drh -Z 9fac49478b578adef606b235c8c223db +P aa02bd3c95e374008b930d296c88dfafaf11c65a +R 767bcf2e23bb391de45c0ed55772662f +U dan +Z 11a6abc8e262f46553fa08a6e318d405 diff --git a/manifest.uuid b/manifest.uuid index f457c0397d..cfc91dee69 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa02bd3c95e374008b930d296c88dfafaf11c65a \ No newline at end of file +737a82444065752785c643b1d29ca097c828effb \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 890ae7c3b2..79c27d06ab 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -290,6 +290,7 @@ struct Stat4Accum { Stat4Sample *aBest; /* Array of nCol best samples */ int iMin; /* Index in a[] of entry with minimum score */ int nSample; /* Current number of samples */ + int nMaxEqZero; /* Max leading 0 in anEq[] for any a[] entry */ int iGet; /* Index of current sample accessed by stat_get() */ Stat4Sample *a; /* Array of mxSample Stat4Sample objects */ sqlite3 *db; /* Database connection, for malloc() */ @@ -551,7 +552,14 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ Stat4Sample *pSample = 0; int i; + /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in Stat4Accum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with Stat4Sample.anEq[m]==0 for (m>=n). */ assert( IsStat4 || nEqZero==0 ); + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } #ifdef SQLITE_ENABLE_STAT4 if( pNew->isPSample==0 ){ @@ -651,12 +659,22 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } } - /* Update the anEq[] fields of any samples already collected. */ + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ for(i=p->nSample-1; i>=0; i--){ int j; - for(j=iChng; jnCol; j++){ - if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); + } + + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } } + p->nMaxEqZero = iChng; } #endif From 24d772cc277cf831ef890495e72c2c9000137b5c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Feb 2017 19:41:16 +0000 Subject: [PATCH 189/292] Move a branch condition in analyze.c inside an #ifdef SQLITE_ENABLE_STAT4 block. FossilOrigin-Name: d6afd98de3ee8b714dfd6477ead955096f623972 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5a5647725b..7980c45c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sANALYZE\swhen\sSQLITE_ENABLE_STAT4\sis\sdefined. -D 2017-02-22T19:27:51.768 +C Move\sa\sbranch\scondition\sin\sanalyze.c\sinside\san\s#ifdef\sSQLITE_ENABLE_STAT4\sblock. +D 2017-02-22T19:41:16.947 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -332,7 +332,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 844359e720d1116592234f2f7938432bdc5b6238 +F src/analyze.c 8a2af8a16e4d95ec2327d3f180cb0bab4b2074c1 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa02bd3c95e374008b930d296c88dfafaf11c65a -R 767bcf2e23bb391de45c0ed55772662f +P 737a82444065752785c643b1d29ca097c828effb +R 36d1036d6c004868a4b3899253e892b2 U dan -Z 11a6abc8e262f46553fa08a6e318d405 +Z b9298378d570f055da46d519ad41197c diff --git a/manifest.uuid b/manifest.uuid index cfc91dee69..69e4a7f8b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -737a82444065752785c643b1d29ca097c828effb \ No newline at end of file +d6afd98de3ee8b714dfd6477ead955096f623972 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 79c27d06ab..098b6c7e37 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -552,16 +552,16 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ Stat4Sample *pSample = 0; int i; + assert( IsStat4 || nEqZero==0 ); + +#ifdef SQLITE_ENABLE_STAT4 /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 ** values in the anEq[] array of any sample in Stat4Accum.a[]. In ** other words, if nMaxEqZero is n, then it is guaranteed that there ** are no samples with Stat4Sample.anEq[m]==0 for (m>=n). */ - assert( IsStat4 || nEqZero==0 ); if( nEqZero>p->nMaxEqZero ){ p->nMaxEqZero = nEqZero; } - -#ifdef SQLITE_ENABLE_STAT4 if( pNew->isPSample==0 ){ Stat4Sample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); From 2ead47cb49d28ead0ea67825bc66a5c360867201 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2017 20:24:10 +0000 Subject: [PATCH 190/292] Change the name of the analyze_as_needed pragma to "optimize". Enhance the comment (which will become documentation, assuming these changes land on trunk) to explain that the optimize pragma is likely to be enhanced in various ways in future releases and that applications should not depend upon the current behavior. FossilOrigin-Name: 9fced545a6f80c55d6dc4a6106cb2d3569566b3e --- manifest | 16 +++++------ manifest.uuid | 2 +- src/pragma.c | 30 ++++++++++++++++----- src/pragma.h | 64 +++++++++++++++++++++++--------------------- tool/mkpragmatab.tcl | 2 +- 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 392b3afc29..508790c7bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sintegrity_check\sand\sother\simprovements\sfrom\strunk. -D 2017-02-22T19:49:54.795 +C Change\sthe\sname\sof\sthe\sanalyze_as_needed\spragma\sto\s"optimize".\s\sEnhance\sthe\ncomment\s(which\swill\sbecome\sdocumentation,\sassuming\sthese\schanges\sland\son\strunk)\nto\sexplain\sthat\sthe\soptimize\spragma\sis\slikely\sto\sbe\senhanced\sin\svarious\sways\nin\sfuture\sreleases\sand\sthat\sapplications\sshould\snot\sdepend\supon\sthe\scurrent\nbehavior. +D 2017-02-22T20:24:10.705 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,8 +388,8 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 97742aae32e645926f6f8fc426169fb5ef97c268 -F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 +F src/pragma.c 46202f2f5ee6957ff5cba581ee3c36507685def0 +F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -1496,7 +1496,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 6fd5ecb4a8debee39e3bb4e63f0634f69edfb861 +F tool/mkpragmatab.tcl 9b499f7301fadeddeae52b95f962ef7e5a718f50 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff213f2ef5bf96754a2264685d25546d8b5ccf0a d6afd98de3ee8b714dfd6477ead955096f623972 -R c12096fa1c0a2878a437d029e47aff9d +P fe073905081b421405ca425ca03c5b8b0ff5f2c8 +R b1ce2ff126387ffc06c8537ddbfa9731 U drh -Z 3c7beae38049bf1fe67e75122f4d0189 +Z 5165d34a54df32aa272e95c6a5e5953c diff --git a/manifest.uuid b/manifest.uuid index 54b7ffb813..8e0845a1c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe073905081b421405ca425ca03c5b8b0ff5f2c8 \ No newline at end of file +9fced545a6f80c55d6dc4a6106cb2d3569566b3e \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 7ff76d02d1..714fb413b6 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1863,16 +1863,29 @@ void sqlite3Pragma( } /* - ** PRAGMA analyze_as_needed - ** PRAGMA schema.analyze_as_needed + ** PRAGMA optimize + ** PRAGMA schema.optimize ** - ** This pragma runs ANALYZE on any tables which would have benefitted - ** from having recent statistics at some point since the start of the - ** current connection. Only tables in "schema" are analyzed in the + ** Attempt to optimize the database. All schemas are optimized in the first + ** form, and only the specified schema is optimized in the second form. + ** + ** The details of optimizations performed by this pragma does are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** Argments to this pragma are currently ignored, but future enhancements + ** might make use of arguments to control which optimizations are allowed + ** or to suggest limits on how much CPU time and I/O should be expended + ** in the optimization effort. + ** + ** The current implementation runs ANALYZE on any tables which might have + ** benefitted from having recent statistics at some point since the start + ** of the current connection. Only tables in "schema" are analyzed in the ** second form. In the first form, all tables except TEMP tables are ** checked. ** - ** A table is analyzed only if both of the following are true: + ** In the current implementation, a table is analyzed only if both of + ** the following are true: ** ** (1) The query planner used sqlite_stat1-style statistics for one or ** more indexes of the table at some point during the lifetime of @@ -1881,8 +1894,11 @@ void sqlite3Pragma( ** (2) One or more indexes of the table are currently unanalyzed OR ** the number of rows in the table has increased by 25 times or more ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. */ - case PragTyp_ANALYZE_AS_NEEDED: { + case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ int iTabCur; /* Cursor for a table whose size needs checking */ HashElem *k; /* Loop over tables of a schema */ diff --git a/src/pragma.h b/src/pragma.h index ba107ec07e..363975df55 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -5,32 +5,32 @@ */ /* The various pragma types */ -#define PragTyp_ANALYZE_AS_NEEDED 0 -#define PragTyp_HEADER_VALUE 1 -#define PragTyp_AUTO_VACUUM 2 -#define PragTyp_FLAG 3 -#define PragTyp_BUSY_TIMEOUT 4 -#define PragTyp_CACHE_SIZE 5 -#define PragTyp_CACHE_SPILL 6 -#define PragTyp_CASE_SENSITIVE_LIKE 7 -#define PragTyp_COLLATION_LIST 8 -#define PragTyp_COMPILE_OPTIONS 9 -#define PragTyp_DATA_STORE_DIRECTORY 10 -#define PragTyp_DATABASE_LIST 11 -#define PragTyp_DEFAULT_CACHE_SIZE 12 -#define PragTyp_ENCODING 13 -#define PragTyp_FOREIGN_KEY_CHECK 14 -#define PragTyp_FOREIGN_KEY_LIST 15 -#define PragTyp_INCREMENTAL_VACUUM 16 -#define PragTyp_INDEX_INFO 17 -#define PragTyp_INDEX_LIST 18 -#define PragTyp_INTEGRITY_CHECK 19 -#define PragTyp_JOURNAL_MODE 20 -#define PragTyp_JOURNAL_SIZE_LIMIT 21 -#define PragTyp_LOCK_PROXY_FILE 22 -#define PragTyp_LOCKING_MODE 23 -#define PragTyp_PAGE_COUNT 24 -#define PragTyp_MMAP_SIZE 25 +#define PragTyp_HEADER_VALUE 0 +#define PragTyp_AUTO_VACUUM 1 +#define PragTyp_FLAG 2 +#define PragTyp_BUSY_TIMEOUT 3 +#define PragTyp_CACHE_SIZE 4 +#define PragTyp_CACHE_SPILL 5 +#define PragTyp_CASE_SENSITIVE_LIKE 6 +#define PragTyp_COLLATION_LIST 7 +#define PragTyp_COMPILE_OPTIONS 8 +#define PragTyp_DATA_STORE_DIRECTORY 9 +#define PragTyp_DATABASE_LIST 10 +#define PragTyp_DEFAULT_CACHE_SIZE 11 +#define PragTyp_ENCODING 12 +#define PragTyp_FOREIGN_KEY_CHECK 13 +#define PragTyp_FOREIGN_KEY_LIST 14 +#define PragTyp_INCREMENTAL_VACUUM 15 +#define PragTyp_INDEX_INFO 16 +#define PragTyp_INDEX_LIST 17 +#define PragTyp_INTEGRITY_CHECK 18 +#define PragTyp_JOURNAL_MODE 19 +#define PragTyp_JOURNAL_SIZE_LIMIT 20 +#define PragTyp_LOCK_PROXY_FILE 21 +#define PragTyp_LOCKING_MODE 22 +#define PragTyp_PAGE_COUNT 23 +#define PragTyp_MMAP_SIZE 24 +#define PragTyp_OPTIMIZE 25 #define PragTyp_PAGE_SIZE 26 #define PragTyp_SECURE_DELETE 27 #define PragTyp_SHRINK_MEMORY 28 @@ -133,11 +133,6 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif - {/* zName: */ "analyze_as_needed", - /* ePragTyp: */ PragTyp_ANALYZE_AS_NEEDED, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -419,6 +414,13 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 2201ea1306..f0ba49b545 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -362,7 +362,7 @@ set pragma_def { NAME: threads FLAG: Result0 - NAME: analyze_as_needed + NAME: optimize FLAG: NoColumns } From bce0414844b9ed5a99c6c2462a92d6db095b9bf2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Feb 2017 00:58:36 +0000 Subject: [PATCH 191/292] Do a single OP_Expire at the very end of "PRAGMA optimize", and omit the OP_Expire on ANALYZE commands invoked by the pragma. FossilOrigin-Name: 188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 30 +++++++++--------------------- src/pragma.c | 1 + src/sqliteInt.h | 1 + src/vdbe.c | 2 ++ 6 files changed, 23 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 508790c7bd..9a9acb5caa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sanalyze_as_needed\spragma\sto\s"optimize".\s\sEnhance\sthe\ncomment\s(which\swill\sbecome\sdocumentation,\sassuming\sthese\schanges\sland\son\strunk)\nto\sexplain\sthat\sthe\soptimize\spragma\sis\slikely\sto\sbe\senhanced\sin\svarious\sways\nin\sfuture\sreleases\sand\sthat\sapplications\sshould\snot\sdepend\supon\sthe\scurrent\nbehavior. -D 2017-02-22T20:24:10.705 +C Do\sa\ssingle\sOP_Expire\sat\sthe\svery\send\sof\s"PRAGMA\soptimize",\sand\somit\sthe\nOP_Expire\son\sANALYZE\scommands\sinvoked\sby\sthe\spragma. +D 2017-02-23T00:58:36.868 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -332,7 +332,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c b4857ec5b46b66049e5a3cec53c071c4902a8e8b +F src/analyze.c 6d8234916c29be943e6ea28b5bef67dff98d9905 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -388,7 +388,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 46202f2f5ee6957ff5cba581ee3c36507685def0 +F src/pragma.c 4b32b014bb4b460bbf0103e4631809428c1ce16b F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -400,7 +400,7 @@ F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h bdc181e371ea618c85f30b4c5ee4d80f4ada6ad7 +F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -462,7 +462,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 37e95d52675bd839cc6c209f6b8d907582a27d44 +F src/vdbe.c f520378e510fd36bbf289921798dbc8f2b3dc30d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe073905081b421405ca425ca03c5b8b0ff5f2c8 -R b1ce2ff126387ffc06c8537ddbfa9731 +P 9fced545a6f80c55d6dc4a6106cb2d3569566b3e +R 5ab35d2cee04a5e705af62d3884bb598 U drh -Z 5165d34a54df32aa272e95c6a5e5953c +Z d17aa4a4e286313b851b7686a8b74646 diff --git a/manifest.uuid b/manifest.uuid index 8e0845a1c4..7b7c25203c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fced545a6f80c55d6dc4a6106cb2d3569566b3e \ No newline at end of file +188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a9893826e2..d1df000943 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1388,27 +1388,14 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1418,10 +1405,11 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* diff --git a/src/pragma.c b/src/pragma.c index 714fb413b6..916ed4aa57 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1940,6 +1940,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); } } + sqlite3VdbeAddOp0(v, OP_Expire); break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fddfe2a57f..e95d63ec35 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1333,6 +1333,7 @@ struct sqlite3 { u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 skipBtreeMutex; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbe.c b/src/vdbe.c index 59e401937f..5901ba0eb5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5523,7 +5523,9 @@ case OP_CreateTable: { /* out2 */ ** Run the SQL statement or statements specified in the P4 string. */ case OP_SqlExec: { + db->nSqlExec++; rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + db->nSqlExec--; if( rc ) goto abort_due_to_error; break; } From 555227bec07e99dddb8b9113d329e48caeba5131 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Feb 2017 02:15:33 +0000 Subject: [PATCH 192/292] Add two NEVER() operators in the sqlite3BtreeRowCountEst() routine. FossilOrigin-Name: 7a959f6d1ea038988cdb4c02d6f37abaec2580a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 9 +++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9a9acb5caa..39c4d8e77a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\sa\ssingle\sOP_Expire\sat\sthe\svery\send\sof\s"PRAGMA\soptimize",\sand\somit\sthe\nOP_Expire\son\sANALYZE\scommands\sinvoked\sby\sthe\spragma. -D 2017-02-23T00:58:36.868 +C Add\stwo\sNEVER()\soperators\sin\sthe\ssqlite3BtreeRowCountEst()\sroutine. +D 2017-02-23T02:15:33.201 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c a4ab1fb5cdeea88c4f76216e41cfecfa505c8c43 +F src/btree.c 19746a7db19308053ff6cb2ac002834822809e54 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 43f903c9082040ced2b421543cb0300c2973647d @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9fced545a6f80c55d6dc4a6106cb2d3569566b3e -R 5ab35d2cee04a5e705af62d3884bb598 +P 188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 +R ea4c0d1070247cd8f7e1aa7ed210345c U drh -Z d17aa4a4e286313b851b7686a8b74646 +Z 01ff8d763fbc7e4c3e39020edd5b116b diff --git a/manifest.uuid b/manifest.uuid index 7b7c25203c..1e1b04c82c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 \ No newline at end of file +7a959f6d1ea038988cdb4c02d6f37abaec2580a0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4d31661f10..788a2d4d03 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5330,8 +5330,13 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - if( pCur->eState!=CURSOR_VALID ) return -1; - if( pCur->apPage[pCur->iPage]->leaf==0 ) return -1; + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + for(n=1, i=0; i<=pCur->iPage; i++){ n *= pCur->apPage[i]->nCell; } From bda4cb876c9636d67a04781aac795b07a39ae965 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Feb 2017 16:30:16 +0000 Subject: [PATCH 193/292] Save a few bytes and a few cycles by setting Vdbe.expmask to zero for statements prepared using legacy interface sqlite3_prepare(). FossilOrigin-Name: a8fd705258643863493476f8b42ee981608a339f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 14 ++++++++------ src/vdbeaux.c | 1 + 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 7980c45c8d..c20fd94d76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sa\sbranch\scondition\sin\sanalyze.c\sinside\san\s#ifdef\sSQLITE_ENABLE_STAT4\sblock. -D 2017-02-22T19:41:16.947 +C Save\sa\sfew\sbytes\sand\sa\sfew\scycles\sby\ssetting\sVdbe.expmask\sto\szero\sfor\nstatements\sprepared\susing\slegacy\sinterface\ssqlite3_prepare(). +D 2017-02-23T16:30:16.521 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -465,8 +465,8 @@ F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 F src/vdbe.c 83f387d9e6842b1dc99f6e85bb577c5bbc4e397d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f -F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 -F src/vdbeaux.c 2f48204a0f2875b098ee046bba9265907297b0b5 +F src/vdbeapi.c 70aabe108c411e529a59d8800445513965334062 +F src/vdbeaux.c 031422c66e272c7f1027070e7f0858f4c418dfbc F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 737a82444065752785c643b1d29ca097c828effb -R 36d1036d6c004868a4b3899253e892b2 +P d6afd98de3ee8b714dfd6477ead955096f623972 +R d82d78a185c578681c4bad1233af7bd3 U dan -Z b9298378d570f055da46d519ad41197c +Z 013ae364526d4d7cd1569b7a9709f5c0 diff --git a/manifest.uuid b/manifest.uuid index 69e4a7f8b9..36ee9348a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6afd98de3ee8b714dfd6477ead955096f623972 \ No newline at end of file +a8fd705258643863493476f8b42ee981608a339f \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 6eb97f1d1d..3d9bcca99f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -154,7 +154,8 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ sqlite3VdbeMemRelease(&p->aVar[i]); p->aVar[i].flags = MEM_Null; } - if( p->isPrepareV2 && p->expmask ){ + assert( p->isPrepareV2 || p->expmask==0 ); + if( p->expmask ){ p->expired = 1; } sqlite3_mutex_leave(mutex); @@ -1258,9 +1259,8 @@ static int vdbeUnbind(Vdbe *p, int i){ ** as if there had been a schema change, on the first sqlite3_step() call ** following any change to the bindings of that parameter. */ - if( p->isPrepareV2 && - ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) - ){ + assert( p->isPrepareV2 || p->expmask==0 ); + if( p->expmask & ((u32)1 << (i&0x001F)) && (i<32 || p->expmask==0xffffffff) ){ p->expired = 1; } return SQLITE_OK; @@ -1523,10 +1523,12 @@ int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ if( pFrom->nVar!=pTo->nVar ){ return SQLITE_ERROR; } - if( pTo->isPrepareV2 && pTo->expmask ){ + assert( pTo->isPrepareV2 || pTo->expmask==0 ); + if( pTo->expmask ){ pTo->expired = 1; } - if( pFrom->isPrepareV2 && pFrom->expmask ){ + assert( pFrom->isPrepareV2 || pFrom->expmask==0 ); + if( pFrom->expmask ){ pFrom->expired = 1; } return sqlite3TransferBindings(pFromStmt, pToStmt); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 292ff2bdd2..67e0f63ef6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -57,6 +57,7 @@ void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ assert( isPrepareV2==1 || isPrepareV2==0 ); if( p==0 ) return; + if( !isPrepareV2 ) p->expmask = 0; #if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) if( !isPrepareV2 ) return; #endif From e6d065a81e6bd7be15c778c3a103d94bf1ad3e0f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Feb 2017 19:58:22 +0000 Subject: [PATCH 194/292] Optimize defragmentPage() in the case where the page contains either one or two free-blocks and a small number of fragmented bytes. FossilOrigin-Name: 202b1c0276aec6b8da64d3277de1ad91c9d62d80 --- manifest | 15 ++++++++----- manifest.uuid | 2 +- src/btree.c | 62 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index c20fd94d76..77cbf1aa41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Save\sa\sfew\sbytes\sand\sa\sfew\scycles\sby\ssetting\sVdbe.expmask\sto\szero\sfor\nstatements\sprepared\susing\slegacy\sinterface\ssqlite3_prepare(). -D 2017-02-23T16:30:16.521 +C Optimize\sdefragmentPage()\sin\sthe\scase\swhere\sthe\spage\scontains\seither\sone\sor\ntwo\sfree-blocks\sand\sa\ssmall\snumber\sof\sfragmented\sbytes. +D 2017-02-24T19:58:22.394 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 +F src/btree.c 1763e0ec3a6cbda48d3a5cb5a7451e46fbc8784d F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f @@ -1557,7 +1557,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d6afd98de3ee8b714dfd6477ead955096f623972 -R d82d78a185c578681c4bad1233af7bd3 +P a8fd705258643863493476f8b42ee981608a339f +R c55510e117c3384aa955b9dc66959921 +T *branch * defragmentpage-opt +T *sym-defragmentpage-opt * +T -sym-trunk * U dan -Z 013ae364526d4d7cd1569b7a9709f5c0 +Z a194b2944e8972170f1e898920de81d2 diff --git a/manifest.uuid b/manifest.uuid index 36ee9348a9..6d120aa8df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8fd705258643863493476f8b42ee981608a339f \ No newline at end of file +202b1c0276aec6b8da64d3277de1ad91c9d62d80 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e78ffef1be..6276a850a0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1317,17 +1317,18 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ /* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. ** ** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a ** b-tree page so that there are no freeblocks or fragment bytes, all ** unused bytes are contained in the unallocated space region, and all ** cells are packed tightly at the end of the page. */ -static int defragmentPage(MemPage *pPage){ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ int i; /* Loop counter */ int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ @@ -1342,7 +1343,6 @@ static int defragmentPage(MemPage *pPage){ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); @@ -1354,10 +1354,44 @@ static int defragmentPage(MemPage *pPage){ cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); + iCellFirst = cellOffset + 2*nCell; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( iFree2 ){ + sz2 = get2byte(&data[iFree2+2]); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + } + cbrk = top+sz; + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddrpBt->usableSize; cbrk = usableSize; - iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; + for(i=0; inFree ){ + return SQLITE_CORRUPT_BKPT; + } + + defragment_out: assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; - data[hdr+7] = 0; memset(&data[iCellFirst], 0, cbrk-iCellFirst); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } return SQLITE_OK; } @@ -1537,7 +1573,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); - rc = defragmentPage(pPage); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); @@ -7689,7 +7725,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0]); + rc = defragmentPage(apNew[0], 0); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) From 3b2ede1f035a86195df4b23c11cd445d6361818e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Feb 2017 16:24:02 +0000 Subject: [PATCH 195/292] Tweak the code on this branch to detect b-tree page corruption in the same cases as the trunk. FossilOrigin-Name: f9863b39d96dce6cb5e49a5f3a445ff3d897a951 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/btree.c | 10 +++++----- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 77cbf1aa41..c6196d9c48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sdefragmentPage()\sin\sthe\scase\swhere\sthe\spage\scontains\seither\sone\sor\ntwo\sfree-blocks\sand\sa\ssmall\snumber\sof\sfragmented\sbytes. -D 2017-02-24T19:58:22.394 +C Tweak\sthe\scode\son\sthis\sbranch\sto\sdetect\sb-tree\spage\scorruption\sin\sthe\ssame\scases\sas\sthe\strunk. +D 2017-02-25T16:24:02.032 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1763e0ec3a6cbda48d3a5cb5a7451e46fbc8784d +F src/btree.c 03149b0f3fec3c1aa3c50a17e997bb442b867632 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f @@ -1557,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8fd705258643863493476f8b42ee981608a339f -R c55510e117c3384aa955b9dc66959921 -T *branch * defragmentpage-opt -T *sym-defragmentpage-opt * -T -sym-trunk * +P 202b1c0276aec6b8da64d3277de1ad91c9d62d80 +R 7a56121f7b6bd4c4a315ae4d10cd07e6 U dan -Z a194b2944e8972170f1e898920de81d2 +Z 0eab51f3ee55bc511deac00d1a02232c diff --git a/manifest.uuid b/manifest.uuid index 6d120aa8df..190b38aa89 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -202b1c0276aec6b8da64d3277de1ad91c9d62d80 \ No newline at end of file +f9863b39d96dce6cb5e49a5f3a445ff3d897a951 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6276a850a0..bf65852fe6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1425,11 +1425,11 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ memcpy(&data[cbrk], &src[pc], size); } data[hdr+7] = 0; - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } defragment_out: + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_BKPT; + } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; @@ -1576,7 +1576,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + assert( gap+2+nByte<=top ); } @@ -7725,7 +7725,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0], 0); + rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) From 70cdf382323dcc97c8eacdad658e58739c6adc2d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Feb 2017 20:57:46 +0000 Subject: [PATCH 196/292] Add an 'extern "C"' block to header file sqlite3userauth.h. FossilOrigin-Name: ffd61fb449a510b2fc90caf86b266733051cc365 --- ext/userauth/sqlite3userauth.h | 8 ++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ext/userauth/sqlite3userauth.h b/ext/userauth/sqlite3userauth.h index 619477cac9..c7d23b9399 100644 --- a/ext/userauth/sqlite3userauth.h +++ b/ext/userauth/sqlite3userauth.h @@ -21,6 +21,10 @@ */ #ifdef SQLITE_USER_AUTHENTICATION +#ifdef __cplusplus +extern "C" { +#endif + /* ** If a database contains the SQLITE_USER table, then the ** sqlite3_user_authenticate() interface must be invoked with an @@ -85,4 +89,8 @@ int sqlite3_user_delete( const char *zUsername /* Username to remove */ ); +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + #endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/manifest b/manifest index ffaf3dbb28..706934adb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sdefragmentPage()\sin\sthe\scase\swhere\sthe\spage\scontains\seither\sone\sor\ntwo\sfree-blocks\sand\sa\ssmall\snumber\sof\sfragmented\sbytes. -D 2017-02-25T17:47:31.387 +C Add\san\s'extern\s"C"'\sblock\sto\sheader\sfile\ssqlite3userauth.h. +D 2017-02-25T20:57:46.730 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -313,7 +313,7 @@ F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc F ext/session/sqlite3session.c 13642d9c754cc18f17e141f82860d269e2adf920 F ext/session/sqlite3session.h d4db650adfcc7a4360e9f12a09c2d117b1db6b53 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 -F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 +F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8fd705258643863493476f8b42ee981608a339f f9863b39d96dce6cb5e49a5f3a445ff3d897a951 -R 7a56121f7b6bd4c4a315ae4d10cd07e6 +P 4cd2a9672c59ea4b3b4cf3d2f139af3c18a8e833 +R 16a6968419b5f6c02a1445afca8164e1 U dan -Z 1f9c94a7513412f3add97a95f1a8ec80 +Z e669ffb28fd5f6d2e359df5fca2d3400 diff --git a/manifest.uuid b/manifest.uuid index 15d62e357c..604dc67407 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cd2a9672c59ea4b3b4cf3d2f139af3c18a8e833 \ No newline at end of file +ffd61fb449a510b2fc90caf86b266733051cc365 \ No newline at end of file From 2efd3488ba12c6d36763a9ef732647c33c56913c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Feb 2017 12:23:52 +0000 Subject: [PATCH 197/292] Remove references to special handling in virtual table methods from the documentation for sqlite3_last_insert_rowid(). FossilOrigin-Name: 660f9569d76e4ff1f5bd4f37f640e6a4fc2cf87d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 10 ++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 706934adb3..1efcbbbcf0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\s'extern\s"C"'\sblock\sto\sheader\sfile\ssqlite3userauth.h. -D 2017-02-25T20:57:46.730 +C Remove\sreferences\sto\sspecial\shandling\sin\svirtual\stable\smethods\sfrom\sthe\ndocumentation\sfor\ssqlite3_last_insert_rowid(). +D 2017-02-27T12:23:52.119 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -397,7 +397,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d -F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 +F src/sqlite.h.in 154b80d215289fb9b845a6f4f9b67ea4fcf049f8 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h a23e18aebdd0d851c2956a74a3a4f12ff202b472 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4cd2a9672c59ea4b3b4cf3d2f139af3c18a8e833 -R 16a6968419b5f6c02a1445afca8164e1 +P ffd61fb449a510b2fc90caf86b266733051cc365 +R 8aa5e89f1f59df441ed610ff04b07915 U dan -Z e669ffb28fd5f6d2e359df5fca2d3400 +Z e6ccfdfcae6209cec3abae13f87bf4f8 diff --git a/manifest.uuid b/manifest.uuid index 604dc67407..d173449ff7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffd61fb449a510b2fc90caf86b266733051cc365 \ No newline at end of file +660f9569d76e4ff1f5bd4f37f640e6a4fc2cf87d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index eaa75fc249..79b9996848 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2048,12 +2048,10 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** have ever occurred on the database connection D, ** then sqlite3_last_insert_rowid(D) returns zero. ** -** ^(If an [INSERT] occurs within a trigger or within a [virtual table] -** method, then this routine will return the [rowid] of the inserted -** row as long as the trigger or virtual table method is running. -** But once the trigger or virtual table method ends, the value returned -** by this routine reverts to what it was before the trigger or virtual -** table method began.)^ +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this From 9c58b63c18938a9bdbcc2f774de45747eec74fbc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Feb 2017 14:52:48 +0000 Subject: [PATCH 198/292] Add an sqlite3_set_last_insert_rowid() method. Use it to work around fts4 and fts5 modifying the last-insert-rowid unintuitively from within commit processing. FossilOrigin-Name: fe41bb5632a5d438acfd682809f1bd12315b970a --- ext/fts3/fts3.c | 5 ++- ext/fts5/fts5_storage.c | 11 +++-- ext/fts5/test/fts5lastrowid.test | 73 ++++++++++++++++++++++++++++++++ manifest | 23 ++++++---- manifest.uuid | 2 +- src/main.c | 15 +++++++ src/sqlite.h.in | 36 +++++++++++++--- test/fts4lastrowid.test | 73 ++++++++++++++++++++++++++++++++ 8 files changed, 217 insertions(+), 21 deletions(-) create mode 100644 ext/fts5/test/fts5lastrowid.test create mode 100644 test/fts4lastrowid.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 7c931c42d4..8af62cda45 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3408,8 +3408,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK && p->nLeafAdd>(nMinMerge/16) && p->nAutoincrmerge && p->nAutoincrmerge!=0xff @@ -3424,6 +3426,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); return rc; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index a695887458..1f8ad2f434 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -1092,12 +1092,17 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ ** Flush any data currently held in-memory to disk. */ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); if( bCommit && p->bTotalsValid ){ - int rc = fts5StorageSaveTotals(p); + rc = fts5StorageSaveTotals(p); p->bTotalsValid = 0; - if( rc!=SQLITE_OK ) return rc; } - return sqlite3Fts5IndexSync(p->pIndex, bCommit); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex, bCommit); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; } int sqlite3Fts5StorageRollback(Fts5Storage *p){ diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test new file mode 100644 index 0000000000..0f7628b233 --- /dev/null +++ b/ext/fts5/test/fts5lastrowid.test @@ -0,0 +1,73 @@ +# 2017 Feb 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the last_insert_rowid functionality with fts5. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5lastrowid + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(str); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + SELECT last_insert_rowid(); +} {3} + +do_execsql_test 1.2 { + BEGIN; + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + COMMIT; + SELECT last_insert_rowid(); +} {6} + +do_execsql_test 1.3 { + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); + SELECT last_insert_rowid(); +} {-22} + +do_execsql_test 1.4 { + BEGIN; + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); + SELECT last_insert_rowid(); + COMMIT; + SELECT last_insert_rowid(); +} {222 222} + +do_execsql_test 1.5 { + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); + INSERT INTO t1 SELECT x FROM x1; + SELECT last_insert_rowid(); +} {226} + +do_execsql_test 1.6 { + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; + SELECT last_insert_rowid(); +} {14} + + +finish_test + diff --git a/manifest b/manifest index 1efcbbbcf0..e571698523 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sreferences\sto\sspecial\shandling\sin\svirtual\stable\smethods\sfrom\sthe\ndocumentation\sfor\ssqlite3_last_insert_rowid(). -D 2017-02-27T12:23:52.119 +C Add\san\ssqlite3_set_last_insert_rowid()\smethod.\sUse\sit\sto\swork\saround\sfts4\sand\nfts5\smodifying\sthe\slast-insert-rowid\sunintuitively\sfrom\swithin\scommit\nprocessing. +D 2017-02-27T14:52:48.644 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c c4d7eecb12de9749851bcab6e5ca616a5803047a +F ext/fts3/fts3.c 95c7041ea75d82d2d9a4cd058904ba889751f5b8 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -106,7 +106,7 @@ F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 -F ext/fts5/fts5_storage.c de0ed8a06738bde433afe11e92295ceaffbc4e58 +F ext/fts5/fts5_storage.c 1db0b6f859ce910027245cf41e8a32c7aaed0042 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be @@ -165,6 +165,7 @@ F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d +F ext/fts5/test/fts5lastrowid.test 4fac1aba696dd6c956e03b0cf91f6f1f3aaec494 F ext/fts5/test/fts5matchinfo.test f7dde99697bcb310ea8faa8eb2714d9f4dfc0e1b F ext/fts5/test/fts5merge.test 9f65f090d214ff865c56bef4f864aaa1182af6e3 F ext/fts5/test/fts5merge2.test a6da3c16d694235938d1939f503cfa53f0943d75 @@ -360,7 +361,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 -F src/main.c e207b81542d13b9f13d61e78ca441f9781f055b0 +F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 F src/malloc.c d0a1474236486165bcb349af82e2a6560178bf7b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb @@ -397,7 +398,7 @@ F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d -F src/sqlite.h.in 154b80d215289fb9b845a6f4f9b67ea4fcf049f8 +F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h a23e18aebdd0d851c2956a74a3a4f12ff202b472 @@ -806,6 +807,7 @@ F test/fts4growth.test e5390da74619cacc389711bac9349640b32c4f9a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d F test/fts4langid.test 65a7332c9bc257919e259a304aa8a38c41655b9d +F test/fts4lastrowid.test fa5e157955a3121615ef3e16ff5196e96c9e1e64 F test/fts4merge.test d2b39f6b1bd4a9738a13540e2d044cba11c43d47 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 @@ -1557,7 +1559,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffd61fb449a510b2fc90caf86b266733051cc365 -R 8aa5e89f1f59df441ed610ff04b07915 +P 660f9569d76e4ff1f5bd4f37f640e6a4fc2cf87d +R 882d86f84a56568be3256ca713764e4e +T *branch * set-last-insert-rowid +T *sym-set-last-insert-rowid * +T -sym-trunk * U dan -Z e6ccfdfcae6209cec3abae13f87bf4f8 +Z 68fc9fa30db7c037a3f29bd13a0c709d diff --git a/manifest.uuid b/manifest.uuid index d173449ff7..ab91f005ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -660f9569d76e4ff1f5bd4f37f640e6a4fc2cf87d \ No newline at end of file +fe41bb5632a5d438acfd682809f1bd12315b970a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 9aad8fdd4c..4ac5327e4f 100644 --- a/src/main.c +++ b/src/main.c @@ -921,6 +921,21 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ return db->lastRowid; } +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + /* ** Return the number of changes in the most recent call to sqlite3_exec(). */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 79b9996848..076a011319 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2040,13 +2040,25 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the -** most recent successful [INSERT] into a rowid table or [virtual table] -** on database connection D. -** ^Inserts into [WITHOUT ROWID] tables are not recorded. -** ^If no successful [INSERT]s into rowid tables -** have ever occurred on the database connection D, -** then sqlite3_last_insert_rowid(D) returns zero. +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. +** +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. ** ** ^(If an [INSERT] occurs within a trigger then this routine will ** return the [rowid] of the inserted row as long as the trigger is @@ -2078,6 +2090,16 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); */ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); + /* ** CAPI3REF: Count The Number Of Rows Modified ** METHOD: sqlite3 diff --git a/test/fts4lastrowid.test b/test/fts4lastrowid.test new file mode 100644 index 0000000000..b5720a2ef3 --- /dev/null +++ b/test/fts4lastrowid.test @@ -0,0 +1,73 @@ +# 2017 Feb 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the last_insert_rowid functionality with fts4. +# + +set testdir [file dirname $argv0] +source [file join [file dirname [info script]] tester.tcl] +set testprefix fts4lastrowid + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts4(str); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + SELECT last_insert_rowid(); +} {3} + +do_execsql_test 1.2 { + BEGIN; + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + COMMIT; + SELECT last_insert_rowid(); +} {6} + +do_execsql_test 1.3 { + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); + SELECT last_insert_rowid(); +} {-22} + +do_execsql_test 1.4 { + BEGIN; + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); + SELECT last_insert_rowid(); + COMMIT; + SELECT last_insert_rowid(); +} {222 222} + +do_execsql_test 1.5 { + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); + INSERT INTO t1 SELECT x FROM x1; + SELECT last_insert_rowid(); +} {226} + +do_execsql_test 1.6 { + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; + SELECT last_insert_rowid(); +} {14} + + +finish_test + From 81415257ec6bdfc97ff5073e7f80bda09abef774 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Feb 2017 17:06:56 +0000 Subject: [PATCH 199/292] Improve performance of fts5 writes by writing the "averages" record once at the end of each transaction instead of every time the table is updated. FossilOrigin-Name: 2b210d691462b463796e7948399133c296e89ebf --- ext/fts5/fts5_storage.c | 14 ++------------ manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 1f8ad2f434..c8c26bd1b6 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -545,11 +545,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ } } - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -753,11 +748,6 @@ int sqlite3Fts5StorageIndexInsert( } sqlite3_free(buf.p); - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -1094,9 +1084,9 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ int rc = SQLITE_OK; i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); - if( bCommit && p->bTotalsValid ){ + if( p->bTotalsValid ){ rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; + if( bCommit ) p->bTotalsValid = 0; } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexSync(p->pIndex, bCommit); diff --git a/manifest b/manifest index 089fbab958..f842047291 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\ssqlite3_set_last_insert_rowid()\smethod.\sUse\sit\sto\swork\saround\sfts4\sand\nfts5\smodifying\sthe\slast-insert-rowid\sunintuitively\sfrom\swithin\scommit\nprocessing. -D 2017-02-27T16:15:29.690 +C Improve\sperformance\sof\sfts5\swrites\sby\swriting\sthe\s"averages"\srecord\sonce\sat\nthe\send\sof\seach\stransaction\sinstead\sof\severy\stime\sthe\stable\sis\supdated. +D 2017-02-27T17:06:56.850 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -106,7 +106,7 @@ F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 -F ext/fts5/fts5_storage.c 1db0b6f859ce910027245cf41e8a32c7aaed0042 +F ext/fts5/fts5_storage.c 8f0e65cb33bde8f449e1c9b4be4600d18b4da6e9 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be @@ -1559,8 +1559,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 660f9569d76e4ff1f5bd4f37f640e6a4fc2cf87d fe41bb5632a5d438acfd682809f1bd12315b970a -R 882d86f84a56568be3256ca713764e4e -T +closed fe41bb5632a5d438acfd682809f1bd12315b970a +P 952a3906b30a818e4574bb85f57150577d04f74e +R 84bac8b8e25067f97464b8201fd688d3 U dan -Z 5437a317a211c2826dc04cbed3f42658 +Z 50f45fcec707e33cc33fa8432e53c9b6 diff --git a/manifest.uuid b/manifest.uuid index c4e0be71d1..a89a664ddf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -952a3906b30a818e4574bb85f57150577d04f74e \ No newline at end of file +2b210d691462b463796e7948399133c296e89ebf \ No newline at end of file From 6e5688d37502f62f7a55bd35c072546d1f78d82e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Feb 2017 17:16:27 +0000 Subject: [PATCH 200/292] Make PATH handling more robust in the batch tools for MSVC. FossilOrigin-Name: 23a8917e848a999533bc66467f7cb2f4f3d45bc1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- tool/GetTclKit.bat | 14 +++++++++++++- tool/build-all-msvc.bat | 6 +++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f842047291..c06c866121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sperformance\sof\sfts5\swrites\sby\swriting\sthe\s"averages"\srecord\sonce\sat\nthe\send\sof\seach\stransaction\sinstead\sof\severy\stime\sthe\stable\sis\supdated. -D 2017-02-27T17:06:56.850 +C Make\sPATH\shandling\smore\srobust\sin\sthe\sbatch\stools\sfor\sMSVC. +D 2017-02-27T17:16:27.906 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -1469,10 +1469,10 @@ F test/wordcount.c 06efb84b7c48a4973c2c24ea06c93d00bce24389 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 -F tool/GetTclKit.bat f94784e3bdc2f50c539266f5467cbf1f27612cb3 +F tool/GetTclKit.bat 6afa640edc7810725aec61c3076ac617c4aaf0b7 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 F tool/addopcodes.tcl 10c889c4a65ec6c5604e4a47306fa77ff57ae189 -F tool/build-all-msvc.bat 018c1b273458a90c8ba633c6f0c5654cfcb138bf x +F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -1559,7 +1559,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 952a3906b30a818e4574bb85f57150577d04f74e -R 84bac8b8e25067f97464b8201fd688d3 -U dan -Z 50f45fcec707e33cc33fa8432e53c9b6 +P 2b210d691462b463796e7948399133c296e89ebf +R 2fadaa28e86ebe3a08aafa06769bc23f +U mistachkin +Z 6b968b4fe877e63444beda0443031271 diff --git a/manifest.uuid b/manifest.uuid index a89a664ddf..af9a30ac81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b210d691462b463796e7948399133c296e89ebf \ No newline at end of file +23a8917e848a999533bc66467f7cb2f4f3d45bc1 \ No newline at end of file diff --git a/tool/GetTclKit.bat b/tool/GetTclKit.bat index 8764b195dd..1c9f92bc12 100644 --- a/tool/GetTclKit.bat +++ b/tool/GetTclKit.bat @@ -124,7 +124,7 @@ IF NOT EXIST "%FRAMEWORKDIR%\csc.exe" ( GOTO errors ) -SET PATH=%FRAMEWORKDIR%;%PATH% +CALL :fn_PrependToPath FRAMEWORKDIR :skip_addToPath @@ -246,6 +246,18 @@ GOTO no_errors ENDLOCAL && SET %1=%VALUE% GOTO :EOF +:fn_PrependToPath + IF NOT DEFINED %1 GOTO :EOF + SETLOCAL + SET __ECHO_CMD=ECHO %%%1%% + FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO ( + SET VALUE=%%V + ) + SET VALUE=%VALUE:"=% + REM " + ENDLOCAL && SET PATH=%VALUE%;%PATH% + GOTO :EOF + :fn_ResetErrorLevel VERIFY > NUL GOTO :EOF diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 497b05e316..aaeb67bdfb 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -460,7 +460,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM REM NOTE: Reset the PATH here to the absolute bare minimum required. REM - SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot% + CALL :fn_ResetPath REM REM NOTE: This is the inner loop. There are normally two iterations, one @@ -818,6 +818,10 @@ GOTO no_errors CALL :fn_ResetErrorLevel GOTO :EOF +:fn_ResetPath + SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot% + GOTO :EOF + :fn_AppendVariable SET __ECHO_CMD=ECHO %%%1%% IF DEFINED %1 ( From 2147221c36ca4f5067e6e2ea0dd0b8eac4a18f37 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Mar 2017 11:30:27 +0000 Subject: [PATCH 201/292] Fix a use-after-free problem in the shell tool code that could occur if an SQL statement were executed after an ".open" command with invalid options. FossilOrigin-Name: ac760db0727209db0a816e112ea5f47e54d54dac --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c06c866121..f194990812 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sPATH\shandling\smore\srobust\sin\sthe\sbatch\stools\sfor\sMSVC. -D 2017-02-27T17:16:27.906 +C Fix\sa\suse-after-free\sproblem\sin\sthe\sshell\stool\scode\sthat\scould\soccur\sif\san\sSQL\nstatement\swere\sexecuted\safter\san\s".open"\scommand\swith\sinvalid\soptions. +D 2017-03-01T11:30:27.730 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -397,7 +397,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d +F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1559,7 +1559,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2b210d691462b463796e7948399133c296e89ebf -R 2fadaa28e86ebe3a08aafa06769bc23f -U mistachkin -Z 6b968b4fe877e63444beda0443031271 +P 23a8917e848a999533bc66467f7cb2f4f3d45bc1 +R 6f5948ee4f1d68e0584e854f4c330a6f +U dan +Z 4914d52d58dfba6bed9b5d6006acef82 diff --git a/manifest.uuid b/manifest.uuid index af9a30ac81..a3f75430ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23a8917e848a999533bc66467f7cb2f4f3d45bc1 \ No newline at end of file +ac760db0727209db0a816e112ea5f47e54d54dac \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index f1c7f3c157..2dcf915f28 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4344,6 +4344,7 @@ static int do_meta_command(char *zLine, ShellState *p){ session_close_all(p); sqlite3_close(p->db); p->db = 0; + p->zDbFilename = 0; sqlite3_free(p->zFreeOnClose); p->zFreeOnClose = 0; /* Check for command-line arguments */ From 76adb239805a543742232090cd4399db3f8f46f6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Mar 2017 13:13:30 +0000 Subject: [PATCH 202/292] Make sure the Vdbe.expmask value is set correctly in sqlite3VdbeSwap(). This fixes a problem introduced by [a8fd7052]. FossilOrigin-Name: 29f54b899e5cf22ece98ab41c39c41d75a4b228d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f194990812..646adca563 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\suse-after-free\sproblem\sin\sthe\sshell\stool\scode\sthat\scould\soccur\sif\san\sSQL\nstatement\swere\sexecuted\safter\san\s".open"\scommand\swith\sinvalid\soptions. -D 2017-03-01T11:30:27.730 +C Make\ssure\sthe\sVdbe.expmask\svalue\sis\sset\scorrectly\sin\ssqlite3VdbeSwap().\nThis\sfixes\sa\sproblem\sintroduced\sby\s[a8fd7052]. +D 2017-03-02T13:13:30.327 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -467,7 +467,7 @@ F src/vdbe.c 83f387d9e6842b1dc99f6e85bb577c5bbc4e397d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 70aabe108c411e529a59d8800445513965334062 -F src/vdbeaux.c 031422c66e272c7f1027070e7f0858f4c418dfbc +F src/vdbeaux.c b632f9151a296c5eb22a2cc955c487ebc2347cb6 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1559,7 +1559,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23a8917e848a999533bc66467f7cb2f4f3d45bc1 -R 6f5948ee4f1d68e0584e854f4c330a6f -U dan -Z 4914d52d58dfba6bed9b5d6006acef82 +P ac760db0727209db0a816e112ea5f47e54d54dac +R be25bb329cc9311f7d92b42176424b6a +U drh +Z eb983447910676b385a2c6496a637326 diff --git a/manifest.uuid b/manifest.uuid index a3f75430ae..9f406e4ffa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac760db0727209db0a816e112ea5f47e54d54dac \ No newline at end of file +29f54b899e5cf22ece98ab41c39c41d75a4b228d \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 67e0f63ef6..8a19c26003 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -86,6 +86,7 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ pA->zSql = pB->zSql; pB->zSql = zTmp; pB->isPrepareV2 = pA->isPrepareV2; + pB->expmask = pA->expmask; } /* From 1cfaf8eafb11a7f2a0ab8dbc47c99dd333667a19 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Mar 2017 14:17:21 +0000 Subject: [PATCH 203/292] Add an optional bitmask of allowed optimizations on the "PRAGMA optimize" command. The 0x01 bit is Debug Mode. FossilOrigin-Name: a35388eef4096c1856b025dbd90143409d4a72d3 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/pragma.c | 60 +++++++++++++++++++++++++++++++++----------- src/pragma.h | 2 +- tool/mkpragmatab.tcl | 2 +- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index bbbb0936a4..fb60f9c239 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\senhancements. -D 2017-03-02T13:22:04.261 +C Add\san\soptional\sbitmask\sof\sallowed\soptimizations\son\sthe\s"PRAGMA\soptimize"\ncommand.\s\sThe\s0x01\sbit\sis\sDebug\sMode. +D 2017-03-02T14:17:21.291 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -389,8 +389,8 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 4b32b014bb4b460bbf0103e4631809428c1ce16b -F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 +F src/pragma.c 75fdb05838c303cf0ce5846ca011b8103531c42c +F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -1498,7 +1498,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 9b499f7301fadeddeae52b95f962ef7e5a718f50 +F tool/mkpragmatab.tcl 2ffe6d5fdc2d3381621d6c77978ba054466e757f F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1560,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a959f6d1ea038988cdb4c02d6f37abaec2580a0 29f54b899e5cf22ece98ab41c39c41d75a4b228d -R 6bb37f552ece3665cee38d0fc37ff06c +P c60cdb47612c05c252613e50a8ac10635469fdfe +R dbee345c7724a1e9049a9032bbb0a60b U drh -Z 8ce562b5567814952b66c349d10ab9fa +Z f9893159b19aa2a55c1ddc2fbced30b3 diff --git a/manifest.uuid b/manifest.uuid index e6cd24f62f..3384381146 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c60cdb47612c05c252613e50a8ac10635469fdfe \ No newline at end of file +a35388eef4096c1856b025dbd90143409d4a72d3 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 916ed4aa57..c5f2a77eef 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1864,34 +1864,51 @@ void sqlite3Pragma( /* ** PRAGMA optimize + ** PRAGMA optimize(MASK) ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) ** ** Attempt to optimize the database. All schemas are optimized in the first - ** form, and only the specified schema is optimized in the second form. + ** two forms, and only the specified schema is optimized in the latter two. ** ** The details of optimizations performed by this pragma does are expected ** to change and improve over time. Applications should anticipate that ** this pragma will perform new optimizations in future releases. ** - ** Argments to this pragma are currently ignored, but future enhancements - ** might make use of arguments to control which optimizations are allowed - ** or to suggest limits on how much CPU time and I/O should be expended - ** in the optimization effort. + ** The optional argument is a bitmask of optimizations to perform: ** - ** The current implementation runs ANALYZE on any tables which might have - ** benefitted from having recent statistics at some point since the start - ** of the current connection. Only tables in "schema" are analyzed in the - ** second form. In the first form, all tables except TEMP tables are - ** checked. + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** In the current implementation, a table is analyzed only if both of + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is 0x000e, which means perform all of the optimizations + ** listed above except do not set Debug Mode. New optimizations may be + ** added in future releases but they will be turned off by default. The + ** default MASK will always be 0x0e. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) The query planner used sqlite_stat1-style statistics for one or + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or ** more indexes of the table at some point during the lifetime of ** the current connection. ** - ** (2) One or more indexes of the table are currently unanalyzed OR + ** (3) One or more indexes of the table are currently unanalyzed OR ** the number of rows in the table has increased by 25 times or more ** since the last time ANALYZE was run. ** @@ -1907,7 +1924,14 @@ void sqlite3Pragma( Index *pIdx; /* An index of the table */ LogEst szThreshold; /* Size threshold above which reanalysis is needd */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xe; + } iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -1932,12 +1956,18 @@ void sqlite3Pragma( if( szThreshold ){ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2, szThreshold); + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", db->aDb[iDb].zDbSName, pTab->zName); - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } } } sqlite3VdbeAddOp0(v, OP_Expire); diff --git a/src/pragma.h b/src/pragma.h index 363975df55..9b1c723b3e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -417,7 +417,7 @@ static const PragmaName aPragmaName[] = { #endif {/* zName: */ "optimize", /* ePragTyp: */ PragTyp_OPTIMIZE, - /* ePragFlg: */ PragFlg_NoColumns, + /* ePragFlg: */ PragFlg_Result1, /* ColNames: */ 0, 0, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index f0ba49b545..59b245cc76 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -363,7 +363,7 @@ set pragma_def { FLAG: Result0 NAME: optimize - FLAG: NoColumns + FLAG: Result1 } # Open the output file From cb1b0a693ac7aef2f05c65dba4d4bf5d78d4cf3b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Mar 2017 14:51:47 +0000 Subject: [PATCH 204/292] When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. FossilOrigin-Name: edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72 --- ext/rbu/rbucrash2.test | 101 +++++++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.c | 12 +++++ manifest | 19 ++++---- manifest.uuid | 2 +- src/test6.c | 17 ++++--- test/tester.tcl | 6 ++- 6 files changed, 139 insertions(+), 18 deletions(-) create mode 100644 ext/rbu/rbucrash2.test diff --git a/ext/rbu/rbucrash2.test b/ext/rbu/rbucrash2.test new file mode 100644 index 0000000000..89c2535a93 --- /dev/null +++ b/ext/rbu/rbucrash2.test @@ -0,0 +1,101 @@ +# 2017 March 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbucrash2 + +db close +forcedelete test.db-oal rbu.db +sqlite3_shutdown +sqlite3_config_uri 1 +reset_db + +# Set up a target database and an rbu update database. The target +# db is the usual "test.db", the rbu db is "test.db2". +# +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0); +} +db_save_and_close + +proc do_rbu_crash_test2 {tn script} { + + foreach f {test.db test.db2} { + set bDone 0 + for {set iDelay 1} {$bDone==0} {incr iDelay} { + forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal + db_restore + + set res [ + crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ + -blocksize 512 {} + ] + + set bDone 1 + if {$res == "1 {child process exited abnormally}"} { + set bDone 0 + } elseif {$res != "0 {}"} { + error "unexected catchsql result: $res" + } + + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + do_execsql_test $tn.delay=$iDelay.f=$f { + PRAGMA integrity_check; + } {ok} + db close + } + } +} + +for {set x 1} {$x < 10} {incr x} { + do_rbu_crash_test2 1.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu savestate + } + rbu close + } +} + +for {set x 1} {$x < 2} {incr x} { + do_rbu_crash_test2 2.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu close + sqlite3rbu rbu test.db test.db2 + } + rbu close + } +} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 48c69115ee..262f96ea3c 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3718,6 +3718,12 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } + /* Sync the db file if currently doing an incremental checkpoint */ + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ @@ -3842,6 +3848,12 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } + /* Sync the db file */ + if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; diff --git a/manifest b/manifest index 646adca563..4ab87a35ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sVdbe.expmask\svalue\sis\sset\scorrectly\sin\ssqlite3VdbeSwap().\nThis\sfixes\sa\sproblem\sintroduced\sby\s[a8fd7052]. -D 2017-03-02T13:13:30.327 +C When\ssaving\sthe\sstate\sof\san\sRBU\supdate\sin\sthe\sincremental-checkpoint\sphase,\nsync\sthe\sdatabase\sfile.\sOtherwise,\sif\sa\spower\sfailure\soccurs\sand\sthe\sRBU\nupdate\sresumed\sfollowing\ssystem\srecovery,\sthe\sdatabase\smay\sbecome\scorrupt. +D 2017-03-02T14:51:47.379 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -250,6 +250,7 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 +F ext/rbu/rbucrash2.test 7f3fe5d6d930be0965f27b49bdcdc8d4a078b73c F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 @@ -262,7 +263,7 @@ F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa -F ext/rbu/sqlite3rbu.c bb0de6cdbdb14a7d55a097238a434b7e99caf318 +F ext/rbu/sqlite3rbu.c 3c2dfd3f27a3ecc5f017ecf82d563b522bbc341a F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -411,7 +412,7 @@ F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d -F src/test6.c 121060d2e79a4f5047eb12b5135b23a6a7a5af01 +F src/test6.c 004ad42f121f693b8cbe060d1a330678abc61620 F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010 F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86 F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 @@ -1173,7 +1174,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 67835ac17e90055f24a9cf52e5c5bce0dd511c74 +F test/tester.tcl 581f0185434daf7026ccede4c07e8d1479186ec5 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1559,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac760db0727209db0a816e112ea5f47e54d54dac -R be25bb329cc9311f7d92b42176424b6a -U drh -Z eb983447910676b385a2c6496a637326 +P 29f54b899e5cf22ece98ab41c39c41d75a4b228d +R fd6b0917cff6359266006d6c00dd0a12 +U dan +Z 1a59570e49a3b7cf84dd8025330375c8 diff --git a/manifest.uuid b/manifest.uuid index 9f406e4ffa..f24ae9a62a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29f54b899e5cf22ece98ab41c39c41d75a4b228d \ No newline at end of file +edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72 \ No newline at end of file diff --git a/src/test6.c b/src/test6.c index a103b9619e..849cdeb3b9 100644 --- a/src/test6.c +++ b/src/test6.c @@ -315,8 +315,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){ assert(pWrite->zBuf); #ifdef TRACE_CRASHTEST - printf("Trashing %d sectors @ %lld (sector %d) (%s)\n", - 1+iLast-iFirst, pWrite->iOffset, iFirst, pWrite->pFile->zName + printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", + 1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize, + pWrite->iOffset, iFirst, pWrite->pFile->zName ); #endif @@ -827,7 +828,7 @@ static int SQLITE_TCLAPI crashNowCmd( } /* -** tclcmd: sqlite_crash_enable ENABLE +** tclcmd: sqlite_crash_enable ENABLE ?DEFAULT? ** ** Parameter ENABLE must be a boolean value. If true, then the "crash" ** vfs is added to the system. If false, it is removed. @@ -839,6 +840,7 @@ static int SQLITE_TCLAPI crashEnableCmd( Tcl_Obj *CONST objv[] ){ int isEnable; + int isDefault = 0; static sqlite3_vfs crashVfs = { 2, /* iVersion */ 0, /* szOsFile */ @@ -862,14 +864,17 @@ static int SQLITE_TCLAPI crashEnableCmd( 0, /* xCurrentTimeInt64 */ }; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "ENABLE"); + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?"); return TCL_ERROR; } if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){ return TCL_ERROR; } + if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){ + return TCL_ERROR; + } if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){ return TCL_OK; @@ -880,7 +885,7 @@ static int SQLITE_TCLAPI crashEnableCmd( crashVfs.mxPathname = pOriginalVfs->mxPathname; crashVfs.pAppData = (void *)pOriginalVfs; crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile; - sqlite3_vfs_register(&crashVfs, 0); + sqlite3_vfs_register(&crashVfs, isDefault); }else{ crashVfs.pAppData = 0; sqlite3_vfs_unregister(&crashVfs); diff --git a/test/tester.tcl b/test/tester.tcl index 1da89fec26..dc6547d033 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1533,6 +1533,7 @@ proc crashsql {args} { set tclbody {} set crashfile "" set dc "" + set dfltvfs 0 set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { @@ -1546,7 +1547,8 @@ proc crashsql {args} { elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ - elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \ + elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\ + elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\ else { error "Unrecognized option: $z" } } @@ -1560,7 +1562,7 @@ proc crashsql {args} { set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] set f [open crash.tcl w] - puts $f "sqlite3_crash_enable 1" + puts $f "sqlite3_crash_enable 1 $dfltvfs" puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile" puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" From 1d62a66230ddf0320d918283141b32bb8a3926ff Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Mar 2017 16:56:48 +0000 Subject: [PATCH 205/292] Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. FossilOrigin-Name: 4012bb3aa91927156ba149caa4e5c622b0729d79 --- ext/rbu/rbucrash2.test | 11 ++++++++--- ext/rbu/rbuprogress.test | 2 ++ ext/rbu/sqlite3rbu.c | 34 +++++++++++++++++++++++++++++++--- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/ext/rbu/rbucrash2.test b/ext/rbu/rbucrash2.test index 89c2535a93..5f2ba604d1 100644 --- a/ext/rbu/rbucrash2.test +++ b/ext/rbu/rbucrash2.test @@ -45,7 +45,12 @@ db_save_and_close proc do_rbu_crash_test2 {tn script} { - foreach f {test.db test.db2} { + foreach {f blksz} { + test.db 512 + test.db2 512 + test.db 4096 + test.db2 4096 + } { set bDone 0 for {set iDelay 1} {$bDone==0} {incr iDelay} { forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal @@ -53,7 +58,7 @@ proc do_rbu_crash_test2 {tn script} { set res [ crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ - -blocksize 512 {} + -blocksize $blksz {} ] set bDone 1 @@ -68,7 +73,7 @@ proc do_rbu_crash_test2 {tn script} { rbu close sqlite3 db test.db - do_execsql_test $tn.delay=$iDelay.f=$f { + do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz { PRAGMA integrity_check; } {ok} db close diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test index af202829c1..078b3b0d30 100644 --- a/ext/rbu/rbuprogress.test +++ b/ext/rbu/rbuprogress.test @@ -40,6 +40,7 @@ proc create_rbu1 {filename} { do_execsql_test 1.0 { + PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); } @@ -266,6 +267,7 @@ foreach bReopen {0 1} { do_test 3.$bReopen.1.0 { reset_db execsql { + PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b); CREATE TABLE t2(a INTEGER PRIMARY KEY, b); CREATE TABLE t3(a INTEGER PRIMARY KEY, b); diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 262f96ea3c..163a544259 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -356,6 +356,7 @@ struct sqlite3rbu { RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; @@ -2620,6 +2621,16 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ p->rc = SQLITE_DONE; p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } } } } @@ -3275,9 +3286,26 @@ int sqlite3rbu_step(sqlite3rbu *p){ p->rc = SQLITE_DONE; } }else{ - RbuFrame *pFrame = &p->aFrame[p->nStep]; - rbuCheckpointFrame(p, pFrame); - p->nStep++; + /* At one point the following block copied a single frame from the + ** wal file to the database file. So that one call to sqlite3rbu_step() + ** checkpointed a single frame. + ** + ** However, if the sector-size is larger than the page-size, and the + ** application calls sqlite3rbu_savestate() or close() immediately + ** after this step, then rbu_step() again, then a power failure occurs, + ** then the database page written here may be damaged. Work around + ** this by checkpointing frames until the next page in the aFrame[] + ** lies on a different disk sector to the current one. */ + u32 iSector; + do{ + RbuFrame *pFrame = &p->aFrame[p->nStep]; + iSector = (pFrame->iDbPage-1) / p->nPagePerSector; + rbuCheckpointFrame(p, pFrame); + p->nStep++; + }while( p->nStepnFrame + && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector) + && p->rc==SQLITE_OK + ); } p->nProgress++; } diff --git a/manifest b/manifest index 4ab87a35ed..ff786aa1dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssaving\sthe\sstate\sof\san\sRBU\supdate\sin\sthe\sincremental-checkpoint\sphase,\nsync\sthe\sdatabase\sfile.\sOtherwise,\sif\sa\spower\sfailure\soccurs\sand\sthe\sRBU\nupdate\sresumed\sfollowing\ssystem\srecovery,\sthe\sdatabase\smay\sbecome\scorrupt. -D 2017-03-02T14:51:47.379 +C Fix\sanother\sRBU\scase\ssimilar\sto\sthe\sprevious.\sThis\sone\sfor\ssystems\swhere\sthe\nsector-size\sis\slarger\sthan\sthe\spage-size. +D 2017-03-02T16:56:48.085 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -250,7 +250,7 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 -F ext/rbu/rbucrash2.test 7f3fe5d6d930be0965f27b49bdcdc8d4a078b73c +F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 @@ -258,12 +258,12 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda -F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b +F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa -F ext/rbu/sqlite3rbu.c 3c2dfd3f27a3ecc5f017ecf82d563b522bbc341a +F ext/rbu/sqlite3rbu.c cba23db39792175295b94ad0086825894b617921 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1560,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29f54b899e5cf22ece98ab41c39c41d75a4b228d -R fd6b0917cff6359266006d6c00dd0a12 +P edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72 +R f72d0ed562c179232e2d4d139b6c6946 U dan -Z 1a59570e49a3b7cf84dd8025330375c8 +Z a85a6fd6f15328765626feb0d9a72fd7 diff --git a/manifest.uuid b/manifest.uuid index f24ae9a62a..b44fc2cacb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72 \ No newline at end of file +4012bb3aa91927156ba149caa4e5c622b0729d79 \ No newline at end of file From 9715f7f033887a2c4ec09592e3668d5fc0d737bd Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Mar 2017 23:40:21 +0000 Subject: [PATCH 206/292] Fix a bug in the 'start of ...' date/time modifiers when they follow a julian day number. Fix for ticket [6097cb92745327a1]. FossilOrigin-Name: 081dbcfb6d82528cefecb36c4491fa6e1a790b17 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/date.c | 1 + test/date.test | 10 ++++++++++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ff786aa1dc..8ace7675fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sRBU\scase\ssimilar\sto\sthe\sprevious.\sThis\sone\sfor\ssystems\swhere\sthe\nsector-size\sis\slarger\sthan\sthe\spage-size. -D 2017-03-02T16:56:48.085 +C Fix\sa\sbug\sin\sthe\s'start\sof\s...'\sdate/time\smodifiers\swhen\sthey\sfollow\sa\njulian\sday\snumber.\s\sFix\sfor\sticket\s[6097cb92745327a1]. +D 2017-03-02T23:40:21.740 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -347,7 +347,7 @@ F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 -F src/date.c dc3f1391d9297f8c748132813aaffcb117090d6e +F src/date.c bb4db348be17903f4bbf2ef3cb208add874deab9 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c 8a29e9b72d4b642189999c41782cd6c5bc43512f @@ -637,7 +637,7 @@ F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b -F test/date.test a6a5a48b90907bca9fbcc79a30be5a715c1ab2fc +F test/date.test e6da60d8ec399de43c076425fbe2eecbc90a31a4 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbselftest.c b2e6cfac59066dbcb7334b66304bb15a5508dd42 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 @@ -1560,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72 -R f72d0ed562c179232e2d4d139b6c6946 -U dan -Z a85a6fd6f15328765626feb0d9a72fd7 +P 4012bb3aa91927156ba149caa4e5c622b0729d79 +R 106d75876c31ad808988639b89376fd6 +U drh +Z d7eafab06f2b8ece511ab9317c8fe2f8 diff --git a/manifest.uuid b/manifest.uuid index b44fc2cacb..9e61a74210 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4012bb3aa91927156ba149caa4e5c622b0729d79 \ No newline at end of file +081dbcfb6d82528cefecb36c4491fa6e1a790b17 \ No newline at end of file diff --git a/src/date.c b/src/date.c index a08248ce2f..6fc66385c5 100644 --- a/src/date.c +++ b/src/date.c @@ -748,6 +748,7 @@ static int parseModifier( p->validHMS = 1; p->h = p->m = 0; p->s = 0.0; + p->rawS = 0; p->validTZ = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ diff --git a/test/date.test b/test/date.test index 2d336e6c00..a9694f18ca 100644 --- a/test/date.test +++ b/test/date.test @@ -596,5 +596,15 @@ datetest 16.29 {datetime(5373484,'-176546 months')} NULL datetest 16.30 {datetime(5373484,'-14712 years')} {-4713-12-31 12:00:00} datetest 16.31 {datetime(5373484,'-14713 years')} NULL +# 2017-03-02: Wrong 'start of day' computation. +# https://www.sqlite.org/src/info/6097cb92745327a1 +# +datetest 17.1 {datetime(2457754, 'start of day')} {2016-12-31 00:00:00} +datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00} +datetest 17.3 {datetime(2457828,'start of day')} {2017-03-15 00:00:00} +datetest 17.4 {datetime(2457828,'start of month')} {2017-03-01 00:00:00} +datetest 17.5 {datetime(2457828,'start of year')} {2017-01-01 00:00:00} + + finish_test From 0e9b43ff853afe97b416e4bd3866f0d27ea985d8 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Mar 2017 16:51:46 +0000 Subject: [PATCH 207/292] Before beginning an incremental checkpoint in RBU, sync the directory containing the target database file. This ensures that the new directory entry created by renaming the *-oal file to *-wal is synced to disk. FossilOrigin-Name: 915a9a28783fbb2f4c0794eb4264ce8c0b9d42f7 --- ext/rbu/sqlite3rbu.c | 7 +++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 163a544259..7dd232b4b5 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2624,6 +2624,7 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ }else{ int nSectorSize; sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; assert( p->nPagePerSector==0 ); nSectorSize = pDb->pMethods->xSectorSize(pDb); if( nSectorSize>p->pgsz ){ @@ -2631,6 +2632,12 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ }else{ p->nPagePerSector = 1; } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); } } } diff --git a/manifest b/manifest index 8ace7675fd..b06ed90284 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\s'start\sof\s...'\sdate/time\smodifiers\swhen\sthey\sfollow\sa\njulian\sday\snumber.\s\sFix\sfor\sticket\s[6097cb92745327a1]. -D 2017-03-02T23:40:21.740 +C Before\sbeginning\san\sincremental\scheckpoint\sin\sRBU,\ssync\sthe\sdirectory\ncontaining\sthe\starget\sdatabase\sfile.\sThis\sensures\sthat\sthe\snew\sdirectory\sentry\ncreated\sby\srenaming\sthe\s*-oal\sfile\sto\s*-wal\sis\ssynced\sto\sdisk. +D 2017-03-03T16:51:46.903 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -263,7 +263,7 @@ F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa -F ext/rbu/sqlite3rbu.c cba23db39792175295b94ad0086825894b617921 +F ext/rbu/sqlite3rbu.c 2a89efba9eeba8e6c89a498dc195e8efbdde2694 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1560,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4012bb3aa91927156ba149caa4e5c622b0729d79 -R 106d75876c31ad808988639b89376fd6 -U drh -Z d7eafab06f2b8ece511ab9317c8fe2f8 +P 081dbcfb6d82528cefecb36c4491fa6e1a790b17 +R c49d10bdecee819ceb83b85a5e3f8244 +U dan +Z 0d48f0e17be43a572b386c16c35f7b42 diff --git a/manifest.uuid b/manifest.uuid index 9e61a74210..91442ae413 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -081dbcfb6d82528cefecb36c4491fa6e1a790b17 \ No newline at end of file +915a9a28783fbb2f4c0794eb4264ce8c0b9d42f7 \ No newline at end of file From 30741eb0d3fb13221aab418d6c8c8922308be290 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Mar 2017 20:02:53 +0000 Subject: [PATCH 208/292] Fix a case introduced by [4cd2a967] where a corrupt database could cause a buffer overwrite. FossilOrigin-Name: 5d0455fece514552ad7f283d56526f53d7c688bd --- manifest | 13 +++--- manifest.uuid | 2 +- src/btree.c | 17 ++++++- test/corruptK.test | 113 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 test/corruptK.test diff --git a/manifest b/manifest index b06ed90284..05070243d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Before\sbeginning\san\sincremental\scheckpoint\sin\sRBU,\ssync\sthe\sdirectory\ncontaining\sthe\starget\sdatabase\sfile.\sThis\sensures\sthat\sthe\snew\sdirectory\sentry\ncreated\sby\srenaming\sthe\s*-oal\sfile\sto\s*-wal\sis\ssynced\sto\sdisk. -D 2017-03-03T16:51:46.903 +C Fix\sa\scase\sintroduced\sby\s[4cd2a967]\swhere\sa\scorrupt\sdatabase\scould\scause\sa\sbuffer\soverwrite. +D 2017-03-03T20:02:53.439 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -340,7 +340,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 03149b0f3fec3c1aa3c50a17e997bb442b867632 +F src/btree.c 91baade79832becb44d85cd7c39b5ebe170666d8 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f @@ -618,6 +618,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 +F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -1560,7 +1561,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 081dbcfb6d82528cefecb36c4491fa6e1a790b17 -R c49d10bdecee819ceb83b85a5e3f8244 +P 915a9a28783fbb2f4c0794eb4264ce8c0b9d42f7 +R 87fee79fc77f11db6d08a83e962c36d1 U dan -Z 0d48f0e17be43a572b386c16c35f7b42 +Z d96b298a9dab6e01e07206495b42117b diff --git a/manifest.uuid b/manifest.uuid index 91442ae413..a7791d4715 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -915a9a28783fbb2f4c0794eb4264ce8c0b9d42f7 \ No newline at end of file +5d0455fece514552ad7f283d56526f53d7c688bd \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index bf65852fe6..e942dbce03 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1355,6 +1355,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; /* This block handles pages with two or fewer free blocks and nMaxFrag ** or fewer fragmented bytes. In this case it is faster to move the @@ -1365,6 +1366,17 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ int iFree = get2byte(&data[hdr+1]); if( iFree ){ int iFree2 = get2byte(&data[iFree]); + + /* pageFindSlot() has already verified that free blocks are sorted + ** in order of offset within the page, and that no block extends + ** past the end of the page. Provided the two free slots do not + ** overlap, this guarantees that the memmove() calls below will not + ** overwrite the usableSize byte buffer, even if the database page + ** is corrupt. */ + assert( iFree2==0 || iFree2>iFree ); + assert( iFree+get2byte(&data[iFree+2]) <= usableSize ); + assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize ); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ u8 *pEnd = &data[cellOffset + nCell*2]; u8 *pAddr; @@ -1372,11 +1384,14 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ int sz = get2byte(&data[iFree+2]); int top = get2byte(&data[hdr+5]); if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_BKPT; sz2 = get2byte(&data[iFree2+2]); + assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; } cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); memmove(&data[cbrk], &data[top], iFree-top); for(pAddr=&data[cellOffset]; pAddrpBt->usableSize; cbrk = usableSize; iCellLast = usableSize - 4; - for(i=0; i Date: Fri, 3 Mar 2017 20:43:43 +0000 Subject: [PATCH 209/292] Fix another corner-case for the 'start of ...' modifier in the date/time functions. Related to ticket [6097cb92745327a1]. FossilOrigin-Name: 8831f4393dda42b3434e7767968caea84bbca2af --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/date.c | 1 + test/date.test | 2 ++ 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 05070243d5..30c7196c3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sintroduced\sby\s[4cd2a967]\swhere\sa\scorrupt\sdatabase\scould\scause\sa\sbuffer\soverwrite. -D 2017-03-03T20:02:53.439 +C Fix\sanother\scorner-case\sfor\sthe\s'start\sof\s...'\smodifier\sin\sthe\sdate/time\nfunctions.\s\sRelated\sto\sticket\s[6097cb92745327a1]. +D 2017-03-03T20:43:43.649 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -347,7 +347,7 @@ F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 -F src/date.c bb4db348be17903f4bbf2ef3cb208add874deab9 +F src/date.c 499343d9cd97fb0c13ec299dbd1ff3d787b1cdac F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c 8a29e9b72d4b642189999c41782cd6c5bc43512f @@ -638,7 +638,7 @@ F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b -F test/date.test e6da60d8ec399de43c076425fbe2eecbc90a31a4 +F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbselftest.c b2e6cfac59066dbcb7334b66304bb15a5508dd42 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 @@ -1561,7 +1561,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 915a9a28783fbb2f4c0794eb4264ce8c0b9d42f7 -R 87fee79fc77f11db6d08a83e962c36d1 -U dan -Z d96b298a9dab6e01e07206495b42117b +P 5d0455fece514552ad7f283d56526f53d7c688bd +R 7ede60c220c81f4f24b446e38f3b85e3 +U drh +Z 2f7d593b5303445a6b7dc1a2027118f1 diff --git a/manifest.uuid b/manifest.uuid index a7791d4715..74049a1b57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d0455fece514552ad7f283d56526f53d7c688bd \ No newline at end of file +8831f4393dda42b3434e7767968caea84bbca2af \ No newline at end of file diff --git a/src/date.c b/src/date.c index 6fc66385c5..c3ce4248c2 100644 --- a/src/date.c +++ b/src/date.c @@ -743,6 +743,7 @@ static int parseModifier( ** or month or year. */ if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); p->validHMS = 1; diff --git a/test/date.test b/test/date.test index a9694f18ca..f002334d17 100644 --- a/test/date.test +++ b/test/date.test @@ -604,6 +604,8 @@ datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00} datetest 17.3 {datetime(2457828,'start of day')} {2017-03-15 00:00:00} datetest 17.4 {datetime(2457828,'start of month')} {2017-03-01 00:00:00} datetest 17.5 {datetime(2457828,'start of year')} {2017-01-01 00:00:00} +datetest 17.6 {datetime(37,'start of year')} NULL +datetest 17.7 {datetime(38,'start of year')} {-4712-01-01 00:00:00} From f3f883fcd735b65bdf5f29790512276e5d7746ba Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Mar 2017 21:36:26 +0000 Subject: [PATCH 210/292] Remove an redundant function call from the date/time function implementation. FossilOrigin-Name: 4a04c48a311b19ba5e566877dc5baff543c41aba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/date.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 30c7196c3f..7865de9c71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\scorner-case\sfor\sthe\s'start\sof\s...'\smodifier\sin\sthe\sdate/time\nfunctions.\s\sRelated\sto\sticket\s[6097cb92745327a1]. -D 2017-03-03T20:43:43.649 +C Remove\san\sredundant\sfunction\scall\sfrom\sthe\sdate/time\sfunction\simplementation. +D 2017-03-03T21:36:26.906 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -347,7 +347,7 @@ F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 -F src/date.c 499343d9cd97fb0c13ec299dbd1ff3d787b1cdac +F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c 8a29e9b72d4b642189999c41782cd6c5bc43512f @@ -1561,7 +1561,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d0455fece514552ad7f283d56526f53d7c688bd -R 7ede60c220c81f4f24b446e38f3b85e3 +P 8831f4393dda42b3434e7767968caea84bbca2af +R d8b3a18d4fd999381a3209784f6b834e U drh -Z 2f7d593b5303445a6b7dc1a2027118f1 +Z 55613e922149954b91d38c0de9685f66 diff --git a/manifest.uuid b/manifest.uuid index 74049a1b57..637cd06ca0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8831f4393dda42b3434e7767968caea84bbca2af \ No newline at end of file +4a04c48a311b19ba5e566877dc5baff543c41aba \ No newline at end of file diff --git a/src/date.c b/src/date.c index c3ce4248c2..b49d506bc4 100644 --- a/src/date.c +++ b/src/date.c @@ -756,7 +756,6 @@ static int parseModifier( p->D = 1; rc = 0; }else if( sqlite3_stricmp(z,"year")==0 ){ - computeYMD(p); p->M = 1; p->D = 1; rc = 0; From 2996796dcf2a01df4cacf94f3c8efd430db9a82d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Mar 2017 21:51:40 +0000 Subject: [PATCH 211/292] If a reprepare is needed after binding to a variable with a number larger than 32, set only the high-order bit of the Vdbe.expmask rather than setting all bits. This could potentially result in fewer false-positive reprepares. FossilOrigin-Name: 45797feefe90cb7da53256b0c42fdaa1221d0a27 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 2 +- src/vdbeaux.c | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7865de9c71..084b50ff08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sredundant\sfunction\scall\sfrom\sthe\sdate/time\sfunction\simplementation. -D 2017-03-03T21:36:26.906 +C If\sa\sreprepare\sis\sneeded\safter\sbinding\sto\sa\svariable\swith\sa\snumber\slarger\nthan\s32,\sset\sonly\sthe\shigh-order\sbit\sof\sthe\sVdbe.expmask\srather\sthan\ssetting\nall\sbits.\s\sThis\scould\spotentially\sresult\sin\sfewer\sfalse-positive\sreprepares. +D 2017-03-03T21:51:40.099 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -467,8 +467,8 @@ F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 F src/vdbe.c 83f387d9e6842b1dc99f6e85bb577c5bbc4e397d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f -F src/vdbeapi.c 70aabe108c411e529a59d8800445513965334062 -F src/vdbeaux.c b632f9151a296c5eb22a2cc955c487ebc2347cb6 +F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 +F src/vdbeaux.c 57361f2e761d92a254638bdbfc03fc68ae6aebc6 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1561,7 +1561,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8831f4393dda42b3434e7767968caea84bbca2af -R d8b3a18d4fd999381a3209784f6b834e +P 4a04c48a311b19ba5e566877dc5baff543c41aba +R 637ce5713001d3d9ffad62c9c4caff88 U drh -Z 55613e922149954b91d38c0de9685f66 +Z 435b5c3ba7edfd2dba5c4b61417da7dd diff --git a/manifest.uuid b/manifest.uuid index 637cd06ca0..9ee7804d61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a04c48a311b19ba5e566877dc5baff543c41aba \ No newline at end of file +45797feefe90cb7da53256b0c42fdaa1221d0a27 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 3d9bcca99f..1f2699dc4d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1260,7 +1260,7 @@ static int vdbeUnbind(Vdbe *p, int i){ ** following any change to the bindings of that parameter. */ assert( p->isPrepareV2 || p->expmask==0 ); - if( p->expmask & ((u32)1 << (i&0x001F)) && (i<32 || p->expmask==0xffffffff) ){ + if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1; } return SQLITE_OK; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8a19c26003..ab4aad7a0f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -4552,8 +4552,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ */ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - if( iVar>32 ){ - v->expmask = 0xffffffff; + if( iVar>=32 ){ + v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } From e1f1c0802cb9644cdb23df28a7b76fe71095f492 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Mar 2017 20:44:13 +0000 Subject: [PATCH 212/292] Fix a typo in a comment. No changes to code. FossilOrigin-Name: ec529bf11b16c801ea438e57d208ff7e4cedf1f9 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/pragma.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6e1234fc69..98be82eb72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinitial\simplementation\sof\sthe\s"PRAGMA\soptimize"\scommand. -D 2017-03-06T17:33:58.293 +C Fix\sa\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2017-03-06T20:44:13.809 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -390,7 +390,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 75fdb05838c303cf0ce5846ca011b8103531c42c +F src/pragma.c d4aef42083d2d60760a736f758276638b976b702 F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -1562,8 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 45797feefe90cb7da53256b0c42fdaa1221d0a27 5f7fc79aa06ca9b79664c50c3c277c98a74ff9a0 -R d076238fa06701bd917c02e95912d525 -T +closed 5f7fc79aa06ca9b79664c50c3c277c98a74ff9a0 +P 137aeb2b160888100bc1e871b00860149e5f6196 +R f69eaa0284279e87544348aa447b8b6c U drh -Z 60608bb3fd59af1c228de2bc78db17f1 +Z 0247be1d8d37c820f7663badba92934a diff --git a/manifest.uuid b/manifest.uuid index c68e3d13a8..75989adcda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -137aeb2b160888100bc1e871b00860149e5f6196 \ No newline at end of file +ec529bf11b16c801ea438e57d208ff7e4cedf1f9 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index c5f2a77eef..aa7ebc8793 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1871,7 +1871,7 @@ void sqlite3Pragma( ** Attempt to optimize the database. All schemas are optimized in the first ** two forms, and only the specified schema is optimized in the latter two. ** - ** The details of optimizations performed by this pragma does are expected + ** The details of optimizations performed by this pragma are expected ** to change and improve over time. Applications should anticipate that ** this pragma will perform new optimizations in future releases. ** From 59dbe3a5efcd5ea47ef6038a861dc4d76838a43f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Mar 2017 23:51:16 +0000 Subject: [PATCH 213/292] Make the default MASK argument for "PRAGMA optimize" be 0xfffe, to allow for future expansion of up to 14 new default-on optimizations. FossilOrigin-Name: 73019a8bba29fd07f73559cd00d5346fa822b439 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 98be82eb72..6a2079f2af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2017-03-06T20:44:13.809 +C Make\sthe\sdefault\sMASK\sargument\sfor\s"PRAGMA\soptimize"\sbe\s0xfffe,\sto\sallow\sfor\nfuture\sexpansion\sof\sup\sto\s14\snew\sdefault-on\soptimizations. +D 2017-03-06T23:51:16.804 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -390,7 +390,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c d4aef42083d2d60760a736f758276638b976b702 +F src/pragma.c b7bf4f1dd9cdcdd3f72f0bdbce3502d69d0f48ba F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 137aeb2b160888100bc1e871b00860149e5f6196 -R f69eaa0284279e87544348aa447b8b6c +P ec529bf11b16c801ea438e57d208ff7e4cedf1f9 +R 299bd0d883e4720337a781b64978238c U drh -Z 0247be1d8d37c820f7663badba92934a +Z 84b44db3e09ed739373a73b84fb9456e diff --git a/manifest.uuid b/manifest.uuid index 75989adcda..94998487fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec529bf11b16c801ea438e57d208ff7e4cedf1f9 \ No newline at end of file +73019a8bba29fd07f73559cd00d5346fa822b439 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index aa7ebc8793..2901f4bdc5 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1892,10 +1892,10 @@ void sqlite3Pragma( ** 0x0008 (Not yet implemented) Create indexes that might have ** been helpful to recent queries ** - ** The default MASK is 0x000e, which means perform all of the optimizations - ** listed above except do not set Debug Mode. New optimizations may be - ** added in future releases but they will be turned off by default. The - ** default MASK will always be 0x0e. + ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all ** of the optimizations listed above except Debug Mode, including new + ** optimizations that have not yet been invented. If new optimizations are + ** ever added that should be off by default, those off-by-default + ** optimizations will have bitmasks of 0x10000 or larger. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** @@ -1930,7 +1930,7 @@ void sqlite3Pragma( opMask = (u32)sqlite3Atoi(zRight); if( (opMask & 0x02)==0 ) break; }else{ - opMask = 0xe; + opMask = 0xfffe; } iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ From 3cf48e3e89ff34d61436751d5a47f15eba570b8a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Mar 2017 03:25:52 +0000 Subject: [PATCH 214/292] Small size reduction and performance increase in the name resolver routine for expressions. FossilOrigin-Name: 1a3554e1d71b666325ff377fae5329d79ce5c05f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/parse.y | 2 +- src/resolve.c | 39 ++++++++++++++++++++++----------------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 6a2079f2af..7a5b12b81f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sdefault\sMASK\sargument\sfor\s"PRAGMA\soptimize"\sbe\s0xfffe,\sto\sallow\sfor\nfuture\sexpansion\sof\sup\sto\s14\snew\sdefault-on\soptimizations. -D 2017-03-06T23:51:16.804 +C Small\ssize\sreduction\sand\sperformance\sincrease\sin\sthe\sname\sresolver\sroutine\nfor\sexpressions. +D 2017-03-07T03:25:52.677 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -386,7 +386,7 @@ F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 +F src/parse.y 48b03113704ee8bd78ee6996d81de7fbee22e105 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 @@ -395,7 +395,7 @@ F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c f9bc0de45a30a450da47b3766de00be89bf9be79 +F src/resolve.c 3fd6fdb3666f24bbcc15039031d0b19a5e53c901 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec529bf11b16c801ea438e57d208ff7e4cedf1f9 -R 299bd0d883e4720337a781b64978238c +P 73019a8bba29fd07f73559cd00d5346fa822b439 +R 1209553f688221f7757a0938ef904e6c U drh -Z 84b44db3e09ed739373a73b84fb9456e +Z 582428290c16e3af0ac541cc18ae2bc9 diff --git a/manifest.uuid b/manifest.uuid index 94998487fc..7407f18c83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73019a8bba29fd07f73559cd00d5346fa822b439 \ No newline at end of file +1a3554e1d71b666325ff377fae5329d79ce5c05f \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 0949e97d16..ef9d3dd0ec 100644 --- a/src/parse.y +++ b/src/parse.y @@ -899,7 +899,7 @@ term(A) ::= INTEGER(X). { A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); A.zStart = X.z; A.zEnd = X.z + X.n; - if( A.pExpr ) A.pExpr->flags |= EP_Leaf; + if( A.pExpr ) A.pExpr->flags |= EP_Leaf|EP_Resolved; } expr(A) ::= VARIABLE(X). { if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){ diff --git a/src/resolve.c b/src/resolve.c index 7d89b6fec5..59157996c5 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -608,33 +608,38 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ - /* A lone identifier is the name of a column. - */ - case TK_ID: { - return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); - } - - /* A table name and column name: ID.ID + /* A column name: ID + ** Or table name and column name: ID.ID ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. */ + case TK_ID: case TK_DOT: { const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; - /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ + if( pExpr->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; + zTable = 0; + zColumn = pExpr->u.zToken; }else{ - assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + zTable = pExpr->pLeft->u.zToken; + zColumn = pRight->u.zToken; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pExpr->pLeft->u.zToken; + zTable = pRight->pLeft->u.zToken; + zColumn = pRight->pRight->u.zToken; + } } return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); } From a92b81f487d8f71ef460817be8f4838e9f13816b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Mar 2017 03:40:48 +0000 Subject: [PATCH 215/292] More size and speed improvements in the expression name resolver. FossilOrigin-Name: e0a3d39f51a50420bae97e2bc9a8c01a5b0d3db8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 13 +++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7a5b12b81f..e3370f8805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\ssize\sreduction\sand\sperformance\sincrease\sin\sthe\sname\sresolver\sroutine\nfor\sexpressions. -D 2017-03-07T03:25:52.677 +C More\ssize\sand\sspeed\simprovements\sin\sthe\sexpression\sname\sresolver. +D 2017-03-07T03:40:48.199 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -395,7 +395,7 @@ F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 3fd6fdb3666f24bbcc15039031d0b19a5e53c901 +F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73019a8bba29fd07f73559cd00d5346fa822b439 -R 1209553f688221f7757a0938ef904e6c +P 1a3554e1d71b666325ff377fae5329d79ce5c05f +R 5deac8a39ddd88b8493847a75a8da44a U drh -Z 582428290c16e3af0ac541cc18ae2bc9 +Z bf228925fa0c3d0f27c143e7173a6889 diff --git a/manifest.uuid b/manifest.uuid index 7407f18c83..9f3f1bbd0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a3554e1d71b666325ff377fae5329d79ce5c05f \ No newline at end of file +e0a3d39f51a50420bae97e2bc9a8c01a5b0d3db8 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 59157996c5..6b3caa625a 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -229,7 +229,8 @@ static int lookupName( } /* Start at the inner-most context and move outward until a match is found */ - while( pNC && cnt==0 ){ + assert( pNC && cnt==0 ); + do{ ExprList *pEList; SrcList *pSrcList = pNC->pSrcList; @@ -414,11 +415,11 @@ static int lookupName( /* Advance to the next name context. The loop will exit when either ** we have a match (cnt>0) or when we run out of name contexts. */ - if( cnt==0 ){ - pNC = pNC->pNext; - nSubquery++; - } - } + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + /* ** If X and Y are NULL (in other words if only the column name Z is From 567bd44917bd5e7553d6a38b6eb51429422f5b37 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Mar 2017 12:18:23 +0000 Subject: [PATCH 216/292] Minor comment typo fixes. No changes to code. FossilOrigin-Name: 01cd9fb176cde30658233e36aad44e4a5519612e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/walker.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e3370f8805..a4690fef93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\ssize\sand\sspeed\simprovements\sin\sthe\sexpression\sname\sresolver. -D 2017-03-07T03:40:48.199 +C Minor\scomment\stypo\sfixes.\s\sNo\schanges\sto\scode. +D 2017-03-07T12:18:23.791 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -477,7 +477,7 @@ F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 -F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 +F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a3554e1d71b666325ff377fae5329d79ce5c05f -R 5deac8a39ddd88b8493847a75a8da44a +P e0a3d39f51a50420bae97e2bc9a8c01a5b0d3db8 +R 3a57497a1a3fd47f649c0fa2a10dc005 U drh -Z bf228925fa0c3d0f27c143e7173a6889 +Z 8979ee765002edd7cbb3fe90b0cd803b diff --git a/manifest.uuid b/manifest.uuid index 9f3f1bbd0a..daf8551ff8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0a3d39f51a50420bae97e2bc9a8c01a5b0d3db8 \ No newline at end of file +01cd9fb176cde30658233e36aad44e4a5519612e \ No newline at end of file diff --git a/src/walker.c b/src/walker.c index d1b1e96a2d..a7123d8ab0 100644 --- a/src/walker.c +++ b/src/walker.c @@ -27,11 +27,11 @@ ** ** WRC_Continue Continue descending down the tree. ** -** WRC_Prune Do not descend into child nodes. But allow +** WRC_Prune Do not descend into child nodes, but allow ** the walk to continue with sibling nodes. ** ** WRC_Abort Do no more callbacks. Unwind the stack and -** return the top-level walk call. +** return from the top-level walk call. ** ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. From 9b38d6660f08c01afc8f6fcc86548ce3167acf21 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Mar 2017 14:38:52 +0000 Subject: [PATCH 217/292] Performance optimization in the tokenizer/parser loop. FossilOrigin-Name: 2cb71583d631cd417acbeebbb4ee950573a9deef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/tokenize.c | 23 ++++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index a4690fef93..11e8d7020b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scomment\stypo\sfixes.\s\sNo\schanges\sto\scode. -D 2017-03-07T12:18:23.791 +C Performance\soptimization\sin\sthe\stokenizer/parser\sloop. +D 2017-03-07T14:38:52.463 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -457,7 +457,7 @@ F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 25ccc63ae2c4163933221f3181c9982b47cd08d2 +F src/tokenize.c dc748c5afcb9e5beb3ef5651bc99a4622e30f6a1 F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0a3d39f51a50420bae97e2bc9a8c01a5b0d3db8 -R 3a57497a1a3fd47f649c0fa2a10dc005 +P 01cd9fb176cde30658233e36aad44e4a5519612e +R 71cd7c04663535bedf446cdfbe42cd2c U drh -Z 8979ee765002edd7cbb3fe90b0cd803b +Z ad71d7ac120da8173e9f5ee4d75c7bc4 diff --git a/manifest.uuid b/manifest.uuid index daf8551ff8..8791cf4146 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01cd9fb176cde30658233e36aad44e4a5519612e \ No newline at end of file +2cb71583d631cd417acbeebbb4ee950573a9deef \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 7bb1179e20..578ddc4f00 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -475,8 +475,8 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ */ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ void *pEngine; /* The LEMON-generated LALR(1) parser */ + int n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ @@ -492,7 +492,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ } pParse->rc = SQLITE_OK; pParse->zTail = zSql; - i = 0; assert( pzErrMsg!=0 ); /* sqlite3ParserTrace(stdout, "parser: "); */ #ifdef sqlite3Parser_ENGINEALWAYSONSTACK @@ -510,12 +509,10 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ assert( pParse->nVar==0 ); assert( pParse->pVList==0 ); while( 1 ){ - assert( i>=0 ); - if( zSql[i]!=0 ){ - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((u8*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ + if( zSql[0]!=0 ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; break; } @@ -529,6 +526,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ }else{ tokenType = TK_SEMI; } + zSql -= n; } if( tokenType>=TK_SPACE ){ assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); @@ -537,18 +535,21 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ break; } if( tokenType==TK_ILLEGAL ){ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", - &pParse->sLastToken); + sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); break; } + zSql += n; }else{ + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); lastTokenParsed = tokenType; + zSql += n; if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } } assert( nErr==0 ); - pParse->zTail = &zSql[i]; + pParse->zTail = zSql; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, From 5cc95ebf37cc7b307e02fd6471d7342af5b24226 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Mar 2017 20:03:25 +0000 Subject: [PATCH 218/292] Add test script ext/rbu/rbu_round_trip.tcl. Uses "dbselftest" to test that "rbu" and "sqldiff" work together. FossilOrigin-Name: 961e79da73b4550b3e5b0f9a617133a76485db67 --- ext/rbu/rbu_round_trip.tcl | 205 +++++++++++++++++++++++++++++++++++++ manifest | 13 +-- manifest.uuid | 2 +- 3 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 ext/rbu/rbu_round_trip.tcl diff --git a/ext/rbu/rbu_round_trip.tcl b/ext/rbu/rbu_round_trip.tcl new file mode 100644 index 0000000000..a4f5d579a1 --- /dev/null +++ b/ext/rbu/rbu_round_trip.tcl @@ -0,0 +1,205 @@ +########################################################################## +# 2016 Mar 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +proc process_cmdline {} { + cmdline::process ::A $::argv { + {make "try to build missing tools"} + {verbose "make more noise"} + database + database2 + } { + This script uses/tests the following tools: + + dbselftest + rbu + sqldiff + sqlite3 + + The user passes the names of two database files - a.db and b.db below - as + arguments. This program: + + 1. Runs [dbselftest --init] against both databases. + 2. Runs [sqldiff --rbu --vtab a.db b.db | sqlite3 .db] to create an + RBU database. + 3. Runs [rbu b.db .db] to patch b.db to a.db. + 4. Runs [sqldiff --table selftest a.db b.db] to check that the selftest + tables are now identical. + 5. Runs [dbselftest] against both databases. + } +} + +########################################################################### +########################################################################### +# Command line options processor. This is generic code that can be copied +# between scripts. +# +namespace eval cmdline { + proc cmdline_error {O E {msg ""}} { + if {$msg != ""} { + puts stderr "Error: $msg" + puts stderr "" + } + + set L [list] + foreach o $O { + if {[llength $o]==1} { + lappend L [string toupper $o] + } + } + + puts stderr "Usage: $::argv0 ?SWITCHES? $L" + puts stderr "" + puts stderr "Switches are:" + foreach o $O { + if {[llength $o]==3} { + foreach {a b c} $o {} + puts stderr [format " -%-15s %s (default \"%s\")" "$a VAL" $c $b] + } elseif {[llength $o]==2} { + foreach {a b} $o {} + puts stderr [format " -%-15s %s" $a $b] + } + } + puts stderr "" + puts stderr $E + exit -1 + } + + proc process {avar lArgs O E} { + upvar $avar A + set zTrailing "" ;# True if ... is present in $O + set lPosargs [list] + + # Populate A() with default values. Also, for each switch in the command + # line spec, set an entry in the idx() array as follows: + # + # {tblname t1 "table name to use"} + # -> [set idx(-tblname) {tblname t1 "table name to use"} + # + # For each position parameter, append its name to $lPosargs. If the ... + # specifier is present, set $zTrailing to the name of the prefix. + # + foreach o $O { + set nm [lindex $o 0] + set nArg [llength $o] + switch -- $nArg { + 1 { + if {[string range $nm end-2 end]=="..."} { + set zTrailing [string range $nm 0 end-3] + } else { + lappend lPosargs $nm + } + } + 2 { + set A($nm) 0 + set idx(-$nm) $o + } + 3 { + set A($nm) [lindex $o 1] + set idx(-$nm) $o + } + default { + error "Error in command line specification" + } + } + } + + # Set explicitly specified option values + # + set nArg [llength $lArgs] + for {set i 0} {$i < $nArg} {incr i} { + set opt [lindex $lArgs $i] + if {[string range $opt 0 0]!="-" || $opt=="--"} break + set c [array names idx "${opt}*"] + if {[llength $c]==0} { cmdline_error $O $E "Unrecognized option: $opt"} + if {[llength $c]>1} { cmdline_error $O $E "Ambiguous option: $opt"} + + if {[llength $idx($c)]==3} { + if {$i==[llength $lArgs]-1} { + cmdline_error $O $E "Option requires argument: $c" + } + incr i + set A([lindex $idx($c) 0]) [lindex $lArgs $i] + } else { + set A([lindex $idx($c) 0]) 1 + } + } + + # Deal with position arguments. + # + set nPosarg [llength $lPosargs] + set nRem [expr $nArg - $i] + if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} { + cmdline_error $O $E + } + for {set j 0} {$j < $nPosarg} {incr j} { + set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]] + } + if {$zTrailing!=""} { + set A($zTrailing) [lrange $lArgs [expr $j+$i] end] + } + } +} ;# namespace eval cmdline +# End of command line options processor. +########################################################################### +########################################################################### + +process_cmdline + +# Check that the specified tool is present. +# +proc check_for_tool {tool} { + if {[file exists $tool]==0 || [file executable $tool]==0} { + puts stderr "Missing $tool... exiting. (run \[make $tool\])" + exit -1 + } +} + +if {$A(make)} { + if {$A(verbose)} { puts "building tools..." } + exec make dbselftest rbu sqlite3 sqldiff +} + +check_for_tool dbselftest +check_for_tool rbu +check_for_tool sqlite3 +check_for_tool sqldiff + +exec ./sqlite3 $A(database) "DROP TABLE selftest;" +exec ./sqlite3 $A(database2) "DROP TABLE selftest;" + +# Run [dbselftest --init] on both databases +if {$A(verbose)} { puts "Running \[dbselftest --init\]" } +exec ./dbselftest --init $A(database) +exec ./dbselftest --init $A(database2) + +# Create an RBU patch. +set tmpname "./rrt-[format %x [expr int(rand()*0x7FFFFFFF)]].db" +if {$A(verbose)} { puts "rbu database is $tmpname" } +exec ./sqldiff --rbu --vtab $A(database2) $A(database) | ./sqlite3 $tmpname + +# Run the [rbu] patch. +if {$A(verbose)} { puts "Running \[rbu]" } +exec ./rbu $A(database2) $tmpname + +set selftest_diff [exec ./sqldiff --table selftest $A(database) $A(database2)] +if {$selftest_diff != ""} { + puts stderr "patching table \"selftest\" failed: $selftest_diff" + exit -1 +} + +# Run [dbselftest] on both databases +if {$A(verbose)} { puts "Running \[dbselftest]" } +exec ./dbselftest $A(database) +exec ./dbselftest $A(database2) + +# Remove the RBU database +file delete $tmpname +puts "round trip test successful." + diff --git a/manifest b/manifest index 11e8d7020b..a3944b0665 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\stokenizer/parser\sloop. -D 2017-03-07T14:38:52.463 +C Add\stest\sscript\sext/rbu/rbu_round_trip.tcl.\sUses\s"dbselftest"\sto\stest\sthat\n"rbu"\sand\s"sqldiff"\swork\stogether. +D 2017-03-07T20:03:25.030 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -249,6 +249,7 @@ F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead +F ext/rbu/rbu_round_trip.tcl f3216836bbaeb10d76a2c70b89c13f59eabbd117 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 @@ -1562,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01cd9fb176cde30658233e36aad44e4a5519612e -R 71cd7c04663535bedf446cdfbe42cd2c -U drh -Z ad71d7ac120da8173e9f5ee4d75c7bc4 +P 2cb71583d631cd417acbeebbb4ee950573a9deef +R ee3baff0fd57ede16b35b4777ef32cf9 +U dan +Z 9c10ed94bada1b1224e7c4aff135f1d6 diff --git a/manifest.uuid b/manifest.uuid index 8791cf4146..744c25a1e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cb71583d631cd417acbeebbb4ee950573a9deef \ No newline at end of file +961e79da73b4550b3e5b0f9a617133a76485db67 \ No newline at end of file From e611f1447132d3d34c7cf83e66a2a5a3abdf518e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 11:44:00 +0000 Subject: [PATCH 219/292] Add the --preserve-rowids option to the ".dump" command in the CLI. FossilOrigin-Name: c60aee24714a47ce12ee2a4dcefb9f55211d3761 --- manifest | 16 +-- manifest.uuid | 2 +- src/shell.c | 352 ++++++++++++++++++++++++++++++++--------------- test/shell1.test | 48 ++++++- 4 files changed, 295 insertions(+), 123 deletions(-) diff --git a/manifest b/manifest index a3944b0665..4a60c5aefa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sscript\sext/rbu/rbu_round_trip.tcl.\sUses\s"dbselftest"\sto\stest\sthat\n"rbu"\sand\s"sqldiff"\swork\stogether. -D 2017-03-07T20:03:25.030 +C Add\sthe\s--preserve-rowids\soption\sto\sthe\s".dump"\scommand\sin\sthe\sCLI. +D 2017-03-08T11:44:00.069 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -399,7 +399,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb +F src/shell.c c42c3031f715712a0cd47d8f08bd2d1dfec8baa0 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1110,7 +1110,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 52ac23a345772ab0d6d3241a21a633fdaa3ed581 +F test/shell1.test 5df739ee2fe5c9dbbb2f9a0158b9723bda0910b8 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2cb71583d631cd417acbeebbb4ee950573a9deef -R ee3baff0fd57ede16b35b4777ef32cf9 -U dan -Z 9c10ed94bada1b1224e7c4aff135f1d6 +P 961e79da73b4550b3e5b0f9a617133a76485db67 +R c754df2de2ef21ae6643ebeaf45e8d82 +U drh +Z bce538a1e35be638681ea14d388ad037 diff --git a/manifest.uuid b/manifest.uuid index 744c25a1e5..7002a8bc75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -961e79da73b4550b3e5b0f9a617133a76485db67 \ No newline at end of file +c60aee24714a47ce12ee2a4dcefb9f55211d3761 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2dcf915f28..07f977d46a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -455,28 +455,6 @@ static int isNumber(const char *z, int *realnum){ return *z==0; } -/* -** A global char* and an SQL function to access its current value -** from within an SQL statement. This program used to use the -** sqlite_exec_printf() API to substitue a string into an SQL statement. -** The correct way to do this with sqlite3 is to use the bind API, but -** since the shell is built around the callback paradigm it would be a lot -** of work. Instead just use this hack, which is quite harmless. -*/ -static const char *zShellStatic = 0; -static void shellstaticFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - assert( 0==argc ); - assert( zShellStatic ); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, zShellStatic, -1, SQLITE_STATIC); -} - - /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -618,6 +596,7 @@ struct ShellState { int countChanges; /* True to display change counts */ int backslashOn; /* Resolve C-style \x escapes in SQL input text */ int outCount; /* Revert to stdout when reaching zero */ + int preserveRowid; /* Preserver ROWID values on a ".dump" command */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ @@ -1333,6 +1312,16 @@ static void set_table_name(ShellState *p, const char *zName){ z[n] = 0; } +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellString ShellString; +struct ShellString { + char *z; + int n; + int nAlloc; +}; + /* zIn is either a pointer to a NULL-terminated string in memory obtained ** from malloc(), or a NULL pointer. The string pointed to by zAppend is ** added to zIn, and the result returned in memory obtained from malloc(). @@ -1341,13 +1330,12 @@ static void set_table_name(ShellState *p, const char *zName){ ** If the third argument, quote, is not '\0', then it is used as a ** quote character for zAppend. */ -static char *appendText(char *zIn, char const *zAppend, char quote){ +static void appendText(ShellString *p, char const *zAppend, char quote){ int len; int i; int nAppend = strlen30(zAppend); - int nIn = (zIn?strlen30(zIn):0); - len = nAppend+nIn+1; + len = nAppend+p->n+1; if( quote ){ len += 2; for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ){ + memset(p, 0, sizeof(*p)); + return; + } } if( quote ){ - char *zCsr = &zIn[nIn]; + char *zCsr = p->z+p->n; *zCsr++ = quote; for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; }else{ - memcpy(&zIn[nIn], zAppend, nAppend); - zIn[len-1] = '\0'; + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; } - - return zIn; } @@ -2024,6 +2015,124 @@ static int shell_exec( return rc; } +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = p->preserveRowid; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ){ + raw_printf(stderr, "Error: out of memory\n"); + exit(1); + } + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + int rc; + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + + + /* ** This is a different callback routine used for dumping the database. @@ -2036,7 +2145,6 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ const char *zTable; const char *zType; const char *zSql; - const char *zPrepStmt = 0; ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azCol); @@ -2046,7 +2154,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zSql = azArg[2]; if( strcmp(zTable, "sqlite_sequence")==0 ){ - zPrepStmt = "DELETE FROM sqlite_sequence;\n"; + raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ raw_printf(p->out, "ANALYZE sqlite_master;\n"); }else if( strncmp(zTable, "sqlite_", 7)==0 ){ @@ -2069,58 +2177,64 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } if( strcmp(zType, "table")==0 ){ - sqlite3_stmt *pTableInfo = 0; - char *zSelect = 0; - char *zTableInfo = 0; - char *zTmp = 0; - int nRow = 0; + ShellString sSelect; + ShellString sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; - zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); - zTableInfo = appendText(zTableInfo, zTable, '"'); - zTableInfo = appendText(zTableInfo, ");", 0); - - rc = sqlite3_prepare_v2(p->db, zTableInfo, -1, &pTableInfo, 0); - free(zTableInfo); - if( rc!=SQLITE_OK || !pTableInfo ){ - return 1; + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; } - zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - zTmp = appendText(zTmp, zTable, '"'); - if( zTmp ){ - zSelect = appendText(zSelect, zTmp, '\''); - free(zTmp); - } - zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); - rc = sqlite3_step(pTableInfo); - while( rc==SQLITE_ROW ){ - const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1); - zSelect = appendText(zSelect, "quote(", 0); - zSelect = appendText(zSelect, zText, '"'); - rc = sqlite3_step(pTableInfo); - if( rc==SQLITE_ROW ){ - zSelect = appendText(zSelect, "), ", 0); - }else{ - zSelect = appendText(zSelect, ") ", 0); + memset(&sTable, 0, sizeof(sTable)); + appendText(&sTable, zTable, '"'); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], '"'); } - nRow++; + appendText(&sTable, ")", 0); } - rc = sqlite3_finalize(pTableInfo); - if( rc!=SQLITE_OK || nRow==0 ){ - free(zSelect); - return 1; - } - zSelect = appendText(zSelect, "|| ')' FROM ", 0); - zSelect = appendText(zSelect, zTable, '"'); - rc = run_table_dump_query(p, zSelect, zPrepStmt); - if( rc==SQLITE_CORRUPT ){ - zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); - run_table_dump_query(p, zSelect, 0); + /* Build an appropriate SELECT statement */ + memset(&sSelect, 0, sizeof(sSelect)); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); } - free(zSelect); + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], 0); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, '"'); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + p->zDestTable = savedDestTable; + p->mode = savedMode; + free(sTable.z); + free(sSelect.z); + if( rc ) p->nErr++; } return 0; } @@ -2425,10 +2539,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); globalDb = p->db; - if( p->db && sqlite3_errcode(p->db)==SQLITE_OK ){ - sqlite3_create_function(p->db, "shellstatic", 0, SQLITE_UTF8, 0, - shellstaticFunc, 0, 0); - } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); @@ -3722,21 +3832,42 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + const char *zLike = 0; + int i; + p->preserveRowid = 0; + for(i=1; ipreserveRowid = 1; + }else + { + raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + rc = 1; + goto meta_command_exit; + } + }else if( zLike ){ + raw_printf(stderr, "Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?\n"); + rc = 1; + goto meta_command_exit; + }else{ + zLike = azArg[i]; + } + } open_db(p, 0); /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - if( nArg!=1 && nArg!=2 ){ - raw_printf(stderr, "Usage: .dump ?LIKE-PATTERN?\n"); - rc = 1; - goto meta_command_exit; - } raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); raw_printf(p->out, "BEGIN TRANSACTION;\n"); p->writableSchema = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_master table is + ** corrupt. */ sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); p->nErr = 0; - if( nArg==1 ){ + if( zLike==0 ){ run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" @@ -3750,21 +3881,20 @@ static int do_meta_command(char *zLine, ShellState *p){ "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 ); }else{ - int i; - for(i=1; iwritableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); @@ -4574,17 +4704,17 @@ static int do_meta_command(char *zLine, ShellState *p){ callback(&data, 1, new_argv, new_colv); rc = SQLITE_OK; }else{ - zShellStatic = azArg[1]; - rc = sqlite3_exec(p->db, + char *zSql; + zSql = sqlite3_mprintf( "SELECT sql FROM " " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE lower(tbl_name) LIKE shellstatic()" + "WHERE lower(tbl_name) LIKE %Q" " AND type!='meta' AND sql NOTNULL " - "ORDER BY rowid", - callback, &data, &zErrMsg); - zShellStatic = 0; + "ORDER BY rowid", azArg[1]); + rc = sqlite3_exec(p->db, zSql, callback, &data, &zErrMsg); + sqlite3_free(zSql); } }else if( nArg==1 ){ rc = sqlite3_exec(p->db, diff --git a/test/shell1.test b/test/shell1.test index 95c3130bdd..0912fa4240 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -300,7 +300,7 @@ do_test shell1-3.4.2 { do_test shell1-3.4.3 { # too many arguments catchcmd "test.db" ".dump FOO BAD" -} {1 {Usage: .dump ?LIKE-PATTERN?}} +} {1 {Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?}} # .echo ON|OFF Turn command echo on or off do_test shell1-3.5.1 { @@ -745,16 +745,58 @@ INSERT INTO "t1" VALUES(''); INSERT INTO "t1" VALUES(1); INSERT INTO "t1" VALUES(2.25); INSERT INTO "t1" VALUES('hello'); -INSERT INTO "t1" VALUES(X'807F'); +INSERT INTO "t1" VALUES(X'807f'); CREATE TABLE t3(x,y); INSERT INTO "t3" VALUES(1,NULL); INSERT INTO "t3" VALUES(2,''); INSERT INTO "t3" VALUES(3,1); INSERT INTO "t3" VALUES(4,2.25); INSERT INTO "t3" VALUES(5,'hello'); -INSERT INTO "t3" VALUES(6,X'807F'); +INSERT INTO "t3" VALUES(6,X'807f'); COMMIT;}} +do_test shell1-4.1.1 { + catchcmd test.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(x); +INSERT INTO "t1"(rowid,"x") VALUES(1,NULL); +INSERT INTO "t1"(rowid,"x") VALUES(2,''); +INSERT INTO "t1"(rowid,"x") VALUES(3,1); +INSERT INTO "t1"(rowid,"x") VALUES(4,2.25); +INSERT INTO "t1"(rowid,"x") VALUES(5,'hello'); +INSERT INTO "t1"(rowid,"x") VALUES(6,X'807f'); +CREATE TABLE t3(x,y); +INSERT INTO "t3"(rowid,"x","y") VALUES(1,1,NULL); +INSERT INTO "t3"(rowid,"x","y") VALUES(2,2,''); +INSERT INTO "t3"(rowid,"x","y") VALUES(3,3,1); +INSERT INTO "t3"(rowid,"x","y") VALUES(4,4,2.25); +INSERT INTO "t3"(rowid,"x","y") VALUES(5,5,'hello'); +INSERT INTO "t3"(rowid,"x","y") VALUES(6,6,X'807f'); +COMMIT;}} + +do_test shell1-4.1.2 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(1,null), (2,''), (3,1), + (4,2.25), (5,'hello'), (6,x'807f'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(x INTEGER PRIMARY KEY, y); +INSERT INTO "t1" VALUES(1,NULL); +INSERT INTO "t1" VALUES(2,''); +INSERT INTO "t1" VALUES(3,1); +INSERT INTO "t1" VALUES(4,2.25); +INSERT INTO "t1" VALUES(5,'hello'); +INSERT INTO "t1" VALUES(6,X'807f'); +COMMIT;}} + + # Test the output of ".mode insert" # do_test shell1-4.2.1 { From f42d31800240fd332de77a7c65e4863dea0dd84a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 12:25:18 +0000 Subject: [PATCH 220/292] In the CLI, avoid unnecessary identifier quoting in the ".dump" output. Also add new ".dump" test cases. FossilOrigin-Name: de65f907610a59e64cbf2214789c11f7117a86a6 --- manifest | 14 +-- manifest.uuid | 2 +- src/shell.c | 217 +++++++++++++++++++++++++++++------------------ test/shell1.test | 131 +++++++++++++++++++++------- 4 files changed, 244 insertions(+), 120 deletions(-) diff --git a/manifest b/manifest index 4a60c5aefa..92484569a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--preserve-rowids\soption\sto\sthe\s".dump"\scommand\sin\sthe\sCLI. -D 2017-03-08T11:44:00.069 +C In\sthe\sCLI,\savoid\sunnecessary\sidentifier\squoting\sin\sthe\s".dump"\soutput.\nAlso\sadd\snew\s".dump"\stest\scases. +D 2017-03-08T12:25:18.771 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -399,7 +399,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c c42c3031f715712a0cd47d8f08bd2d1dfec8baa0 +F src/shell.c d1ba571e8325f727f0c9571079e27b8a4595d6fd F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1110,7 +1110,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 5df739ee2fe5c9dbbb2f9a0158b9723bda0910b8 +F test/shell1.test bc1ca4161e1f459c9cd412d3f21f4ee635cf2322 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 961e79da73b4550b3e5b0f9a617133a76485db67 -R c754df2de2ef21ae6643ebeaf45e8d82 +P c60aee24714a47ce12ee2a4dcefb9f55211d3761 +R b3f6f7a6c1a0fc140ee14b2a6ca3bfe0 U drh -Z bce538a1e35be638681ea14d388ad037 +Z f9eff36eb51a780bc84edce006ee1256 diff --git a/manifest.uuid b/manifest.uuid index 7002a8bc75..775616cedc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c60aee24714a47ce12ee2a4dcefb9f55211d3761 \ No newline at end of file +de65f907610a59e64cbf2214789c11f7117a86a6 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 07f977d46a..321a39f54b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -555,6 +555,123 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellText ShellText; +struct ShellText { + char *z; + int n; + int nAlloc; +}; + +/* +** Initialize and destroy a ShellText object +*/ +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ){ + memset(p, 0, sizeof(*p)); + return; + } + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} #if defined(SQLITE_ENABLE_SESSION) /* @@ -1274,6 +1391,7 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ return shell_callback(pArg, nArg, azArg, azCol, NULL); } + /* ** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the @@ -1281,7 +1399,7 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ */ static void set_table_name(ShellState *p, const char *zName){ int i, n; - int needQuote; + int cQuote; char *z; if( p->zDestTable ){ @@ -1289,86 +1407,24 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - needQuote = !isalpha((unsigned char)*zName) && *zName!='_'; - for(i=n=0; zName[i]; i++, n++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ){ - needQuote = 1; - if( zName[i]=='\'' ) n++; - } - } - if( needQuote ) n += 2; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += 2; z = p->zDestTable = malloc( n+1 ); if( z==0 ){ raw_printf(stderr,"Error: out of memory\n"); exit(1); } n = 0; - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; for(i=0; zName[i]; i++){ z[n++] = zName[i]; - if( zName[i]=='\'' ) z[n++] = '\''; + if( zName[i]==cQuote ) z[n++] = cQuote; } - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; z[n] = 0; } -/* -** A variable length string to which one can append text. -*/ -typedef struct ShellString ShellString; -struct ShellString { - char *z; - int n; - int nAlloc; -}; - -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. -** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. -*/ -static void appendText(ShellString *p, char const *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); - - len = nAppend+p->n+1; - if( quote ){ - len += 2; - for(i=0; in+len>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - if( p->z==0 ){ - memset(p, 0, sizeof(*p)); - return; - } - } - - if( quote ){ - char *zCsr = p->z+p->n; - *zCsr++ = quote; - for(i=0; in = (int)(zCsr - p->z); - *zCsr = '\0'; - }else{ - memcpy(p->z+p->n, zAppend, nAppend); - p->n += nAppend; - p->z[p->n] = '\0'; - } -} - /* ** Execute a query statement that will generate SQL output. Print @@ -2131,9 +2187,6 @@ static char **tableColumnList(ShellState *p, const char *zTab){ return azCol; } - - - /* ** This is a different callback routine used for dumping the database. ** Each row received by this callback consists of a table name, @@ -2177,8 +2230,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } if( strcmp(zType, "table")==0 ){ - ShellString sSelect; - ShellString sTable; + ShellText sSelect; + ShellText sTable; char **azCol; int i; char *savedDestTable; @@ -2192,8 +2245,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - memset(&sTable, 0, sizeof(sTable)); - appendText(&sTable, zTable, '"'); + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); /* If preserving the rowid, add a column list after the table name. ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" ** instead of the usual "INSERT INTO tab VALUES(...)". @@ -2203,27 +2256,27 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ appendText(&sTable, azCol[0], 0); for(i=1; azCol[i]; i++){ appendText(&sTable, ",", 0); - appendText(&sTable, azCol[i], '"'); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); } appendText(&sTable, ")", 0); } /* Build an appropriate SELECT statement */ - memset(&sSelect, 0, sizeof(sSelect)); + initText(&sSelect); appendText(&sSelect, "SELECT ", 0); if( azCol[0] ){ appendText(&sSelect, azCol[0], 0); appendText(&sSelect, ",", 0); } for(i=1; azCol[i]; i++){ - appendText(&sSelect, azCol[i], 0); + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); if( azCol[i+1] ){ appendText(&sSelect, ",", 0); } } freeColumnList(azCol); appendText(&sSelect, " FROM ", 0); - appendText(&sSelect, zTable, '"'); + appendText(&sSelect, zTable, quoteChar(zTable)); savedDestTable = p->zDestTable; savedMode = p->mode; @@ -2232,8 +2285,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); p->zDestTable = savedDestTable; p->mode = savedMode; - free(sTable.z); - free(sSelect.z); + freeText(&sTable); + freeText(&sSelect); if( rc ) p->nErr++; } return 0; diff --git a/test/shell1.test b/test/shell1.test index 0912fa4240..7f282292c0 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -740,41 +740,46 @@ do_test shell1-4.1 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x); -INSERT INTO "t1" VALUES(NULL); -INSERT INTO "t1" VALUES(''); -INSERT INTO "t1" VALUES(1); -INSERT INTO "t1" VALUES(2.25); -INSERT INTO "t1" VALUES('hello'); -INSERT INTO "t1" VALUES(X'807f'); +INSERT INTO t1 VALUES(NULL); +INSERT INTO t1 VALUES(''); +INSERT INTO t1 VALUES(1); +INSERT INTO t1 VALUES(2.25); +INSERT INTO t1 VALUES('hello'); +INSERT INTO t1 VALUES(X'807f'); CREATE TABLE t3(x,y); -INSERT INTO "t3" VALUES(1,NULL); -INSERT INTO "t3" VALUES(2,''); -INSERT INTO "t3" VALUES(3,1); -INSERT INTO "t3" VALUES(4,2.25); -INSERT INTO "t3" VALUES(5,'hello'); -INSERT INTO "t3" VALUES(6,X'807f'); +INSERT INTO t3 VALUES(1,NULL); +INSERT INTO t3 VALUES(2,''); +INSERT INTO t3 VALUES(3,1); +INSERT INTO t3 VALUES(4,2.25); +INSERT INTO t3 VALUES(5,'hello'); +INSERT INTO t3 VALUES(6,X'807f'); COMMIT;}} +# The --preserve-rowids option to .dump +# do_test shell1-4.1.1 { catchcmd test.db {.dump --preserve-rowids} } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x); -INSERT INTO "t1"(rowid,"x") VALUES(1,NULL); -INSERT INTO "t1"(rowid,"x") VALUES(2,''); -INSERT INTO "t1"(rowid,"x") VALUES(3,1); -INSERT INTO "t1"(rowid,"x") VALUES(4,2.25); -INSERT INTO "t1"(rowid,"x") VALUES(5,'hello'); -INSERT INTO "t1"(rowid,"x") VALUES(6,X'807f'); +INSERT INTO t1(rowid,x) VALUES(1,NULL); +INSERT INTO t1(rowid,x) VALUES(2,''); +INSERT INTO t1(rowid,x) VALUES(3,1); +INSERT INTO t1(rowid,x) VALUES(4,2.25); +INSERT INTO t1(rowid,x) VALUES(5,'hello'); +INSERT INTO t1(rowid,x) VALUES(6,X'807f'); CREATE TABLE t3(x,y); -INSERT INTO "t3"(rowid,"x","y") VALUES(1,1,NULL); -INSERT INTO "t3"(rowid,"x","y") VALUES(2,2,''); -INSERT INTO "t3"(rowid,"x","y") VALUES(3,3,1); -INSERT INTO "t3"(rowid,"x","y") VALUES(4,4,2.25); -INSERT INTO "t3"(rowid,"x","y") VALUES(5,5,'hello'); -INSERT INTO "t3"(rowid,"x","y") VALUES(6,6,X'807f'); +INSERT INTO t3(rowid,x,y) VALUES(1,1,NULL); +INSERT INTO t3(rowid,x,y) VALUES(2,2,''); +INSERT INTO t3(rowid,x,y) VALUES(3,3,1); +INSERT INTO t3(rowid,x,y) VALUES(4,4,2.25); +INSERT INTO t3(rowid,x,y) VALUES(5,5,'hello'); +INSERT INTO t3(rowid,x,y) VALUES(6,6,X'807f'); COMMIT;}} +# If the table contains an INTEGER PRIMARY KEY, do not record a separate +# rowid column in the output. +# do_test shell1-4.1.2 { db close forcedelete test2.db @@ -788,14 +793,80 @@ do_test shell1-4.1.2 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x INTEGER PRIMARY KEY, y); -INSERT INTO "t1" VALUES(1,NULL); -INSERT INTO "t1" VALUES(2,''); -INSERT INTO "t1" VALUES(3,1); -INSERT INTO "t1" VALUES(4,2.25); -INSERT INTO "t1" VALUES(5,'hello'); -INSERT INTO "t1" VALUES(6,X'807f'); +INSERT INTO t1 VALUES(1,NULL); +INSERT INTO t1 VALUES(2,''); +INSERT INTO t1 VALUES(3,1); +INSERT INTO t1 VALUES(4,2.25); +INSERT INTO t1 VALUES(5,'hello'); +INSERT INTO t1 VALUES(6,X'807f'); COMMIT;}} +# Verify that the table named [table] is correctly quoted and that +# an INTEGER PRIMARY KEY DESC is not an alias for the rowid. +# +do_test shell1-4.1.3 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE [table](x INTEGER PRIMARY KEY DESC, y); + INSERT INTO [table] VALUES(1,null), (12,''), (23,1), + (34,2.25), (45,'hello'), (56,x'807f'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE [table](x INTEGER PRIMARY KEY DESC, y); +INSERT INTO "table"(rowid,x,y) VALUES(1,1,NULL); +INSERT INTO "table"(rowid,x,y) VALUES(2,12,''); +INSERT INTO "table"(rowid,x,y) VALUES(3,23,1); +INSERT INTO "table"(rowid,x,y) VALUES(4,34,2.25); +INSERT INTO "table"(rowid,x,y) VALUES(5,45,'hello'); +INSERT INTO "table"(rowid,x,y) VALUES(6,56,X'807f'); +COMMIT;}} + +# Do not record rowids for a WITHOUT ROWID table. Also check correct quoting +# of table names that contain odd characters. +# +do_test shell1-4.1.4 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; + INSERT INTO [ta<>ble] VALUES(1,null), (12,''), (23,1), + (34,2.25), (45,'hello'), (56,x'807f'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; +INSERT INTO "ta<>ble" VALUES(1,NULL); +INSERT INTO "ta<>ble" VALUES(12,''); +INSERT INTO "ta<>ble" VALUES(23,1); +INSERT INTO "ta<>ble" VALUES(34,2.25); +INSERT INTO "ta<>ble" VALUES(45,'hello'); +INSERT INTO "ta<>ble" VALUES(56,X'807f'); +COMMIT;}} + +# Do not record rowids if the rowid is inaccessible +# +do_test shell1-4.1.5 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(_ROWID_,rowid,oid); + INSERT INTO t1 VALUES(1,null,'alpha'), (12,'',99), (23,1,x'b0b1b2'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(_ROWID_,rowid,oid); +INSERT INTO t1 VALUES(1,NULL,'alpha'); +INSERT INTO t1 VALUES(12,'',99); +INSERT INTO t1 VALUES(23,1,X'b0b1b2'); +COMMIT;}} # Test the output of ".mode insert" # From acc83d2e4f7bc0f3674b1c761bdcd7dfea41d319 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 13:50:40 +0000 Subject: [PATCH 221/292] Add the shathree.c extension for implementing SHA3() and SHA3_QUERY() SQL functions. FossilOrigin-Name: f7ca9193ddafd3676406bdfeb1b7d21182c2a3c1 --- ext/misc/shathree.c | 703 ++++++++++++++++++++++++++++++++++++++++++++ manifest | 11 +- manifest.uuid | 2 +- 3 files changed, 710 insertions(+), 6 deletions(-) create mode 100644 ext/misc/shathree.c diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c new file mode 100644 index 0000000000..a27faedb3f --- /dev/null +++ b/ext/misc/shathree.c @@ -0,0 +1,703 @@ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_query(Y,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. +** +** The sha3_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +typedef sqlite3_uint64 u64; + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. A single R character occurs before the +** start of each row. N means a NULL value. I mean an 8-byte +** little-endian integer . F is a floating point number with +** an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + return rc; +} diff --git a/manifest b/manifest index 92484569a1..1056796a52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sCLI,\savoid\sunnecessary\sidentifier\squoting\sin\sthe\s".dump"\soutput.\nAlso\sadd\snew\s".dump"\stest\scases. -D 2017-03-08T12:25:18.771 +C Add\sthe\sshathree.c\sextension\sfor\simplementing\sSHA3()\sand\sSHA3_QUERY()\sSQL\nfunctions. +D 2017-03-08T13:50:40.092 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -225,6 +225,7 @@ F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 +F ext/misc/shathree.c 465f508eb588e6389a8403fc9cf1e9924dcbc581 w ext/misc/sha3b.c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1563,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c60aee24714a47ce12ee2a4dcefb9f55211d3761 -R b3f6f7a6c1a0fc140ee14b2a6ca3bfe0 +P de65f907610a59e64cbf2214789c11f7117a86a6 +R 23823589b279aa36324ed0cbad2b7a8d U drh -Z f9eff36eb51a780bc84edce006ee1256 +Z 7be543a90293390e2947c08c40f4663a diff --git a/manifest.uuid b/manifest.uuid index 775616cedc..43cc4100d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de65f907610a59e64cbf2214789c11f7117a86a6 \ No newline at end of file +f7ca9193ddafd3676406bdfeb1b7d21182c2a3c1 \ No newline at end of file From 4146b4e34a29a1710a343f585460f7372ea1af09 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 15:03:12 +0000 Subject: [PATCH 222/292] Improved comments explaining the sha3_query() format. Fix a performance #ifdef error in the sha3 implementation. FossilOrigin-Name: 54ef7abd7f5b16f4b29c9519d283e142c9340fbf --- ext/misc/shathree.c | 19 +++++++++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index a27faedb3f..52bfdb0363 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -442,7 +442,7 @@ static void SHA3Update( } #endif for(; iu.x[p->nLoaded] ^= aData[i]; #elif SHA3_BYTEORDER==4321 p->u.x[p->nLoaded^0x07] ^= aData[i]; @@ -556,12 +556,19 @@ static void hash_step_vformat( ** T: ** ** is the original SQL text for each statement run and is -** the size of that text. A single R character occurs before the -** start of each row. N means a NULL value. I mean an 8-byte -** little-endian integer . F is a floating point number with -** an 8-byte little-endian IEEE floating point value . +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . ** B means blobs of bytes. T means text rendered as -** bytes of UTF-8. +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. */ static void sha3QueryFunc( sqlite3_context *context, diff --git a/manifest b/manifest index 1056796a52..a10092847b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sshathree.c\sextension\sfor\simplementing\sSHA3()\sand\sSHA3_QUERY()\sSQL\nfunctions. -D 2017-03-08T13:50:40.092 +C Improved\scomments\sexplaining\sthe\ssha3_query()\sformat.\s\sFix\sa\sperformance\s#ifdef\nerror\sin\sthe\ssha3\simplementation. +D 2017-03-08T15:03:12.217 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -225,7 +225,7 @@ F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 -F ext/misc/shathree.c 465f508eb588e6389a8403fc9cf1e9924dcbc581 w ext/misc/sha3b.c +F ext/misc/shathree.c 38aa4c3c3d3c3fc34b8ea367bed299d8d8a224b2 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de65f907610a59e64cbf2214789c11f7117a86a6 -R 23823589b279aa36324ed0cbad2b7a8d +P f7ca9193ddafd3676406bdfeb1b7d21182c2a3c1 +R 702b29708c528596102031b1a9ef3803 U drh -Z 7be543a90293390e2947c08c40f4663a +Z d26e763b8a8a90fd62d33b05211ec0e3 diff --git a/manifest.uuid b/manifest.uuid index 43cc4100d3..1a49212972 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7ca9193ddafd3676406bdfeb1b7d21182c2a3c1 \ No newline at end of file +54ef7abd7f5b16f4b29c9519d283e142c9340fbf \ No newline at end of file From 1554bc8a2e7335ef39e39383710894942dcc9f54 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 16:10:34 +0000 Subject: [PATCH 223/292] Add the ".sha3sum" command to the CLI - used to compute a cryptographic hash of the content of a database file or of individual tables with in that file. FossilOrigin-Name: fc663799075a22b0a61a6a114116bb2d1b96d4ab --- manifest | 12 +- manifest.uuid | 2 +- src/shell.c | 751 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 758 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a10092847b..97d4af1fb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sexplaining\sthe\ssha3_query()\sformat.\s\sFix\sa\sperformance\s#ifdef\nerror\sin\sthe\ssha3\simplementation. -D 2017-03-08T15:03:12.217 +C Add\sthe\s".sha3sum"\scommand\sto\sthe\sCLI\s-\sused\sto\scompute\sa\scryptographic\shash\nof\sthe\scontent\sof\sa\sdatabase\sfile\sor\sof\sindividual\stables\swith\sin\nthat\sfile. +D 2017-03-08T16:10:34.827 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c d1ba571e8325f727f0c9571079e27b8a4595d6fd +F src/shell.c 8cebab1fdfb7427216c63a46d9116d31ef06dd55 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7ca9193ddafd3676406bdfeb1b7d21182c2a3c1 -R 702b29708c528596102031b1a9ef3803 +P 54ef7abd7f5b16f4b29c9519d283e142c9340fbf +R 1784a0b42056ce37f7cf91782b133804 U drh -Z d26e763b8a8a90fd62d33b05211ec0e3 +Z 89ed86e3a2610bc780c3304d2a434090 diff --git a/manifest.uuid b/manifest.uuid index 1a49212972..c49a19c8c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54ef7abd7f5b16f4b29c9519d283e142c9340fbf \ No newline at end of file +fc663799075a22b0a61a6a114116bb2d1b96d4ab \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 321a39f54b..06cf36cccc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -673,6 +673,658 @@ static char quoteChar(const char *zName){ return 0; } +/****************************************************************************** +** SHA3 hash implementation copied from ../ext/misc/shathree.c +*/ +typedef sqlite3_uint64 u64; +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} +/* End of SHA3 hashing logic copy/pasted from ../ext/misc/shathree.c +********************************************************************************/ + #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session @@ -2402,6 +3054,7 @@ static char zHelp[] = #if defined(SQLITE_ENABLE_SESSION) ".session CMD ... Create or control sessions\n" #endif + ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" ".stats ?on|off? Show stats or turn stats on or off\n" @@ -2605,6 +3258,14 @@ static void open_db(ShellState *p, int keepAlive){ readfileFunc, 0, 0); sqlite3_create_function(p->db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); + sqlite3_create_function(p->db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); } } @@ -5036,6 +5697,96 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + const char *zLike = 0; /* Which table to checksum. 0 means everything */ + int i; /* Loop counter */ + int bSchema = 0; /* Also hash the schema */ + int iSize = 224; /* Hash algorithm to use */ + int bDebug = 0; /* Only show the query that would have run */ + sqlite3_stmt *pStmt; /* For querying tables names */ + char *zSql; /* SQL to be run */ + ShellText sQuery; /* Set of queries used to read all content */ + for(i=1; i1" + " UNION ALL SELECT 'sqlite_master'" + " ORDER BY 1 collate nocase"; + }else{ + zSql = "SELECT lower(name) FROM sqlite_master" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " AND name NOT LIKE 'sqlite_%'" + " ORDER BY 1 collate nocase"; + } + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + initText(&sQuery); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; + if( strncmp(zTab, "sqlite_",7)!=0 ){ + appendText(&sQuery,"SELECT * FROM ", 0); + appendText(&sQuery,zTab,'"'); + appendText(&sQuery," NOT INDEXED;", 0); + }else if( strcmp(zTab, "sqlite_master")==0 ){ + appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_master" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); + }else if( strcmp(zTab, "sqlite_stat3")==0 + || strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); + } + } + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("SELECT lower(hex(sha3_query(%Q,%d))) AS hash;", + sQuery.z, iSize); + freeText(&sQuery); + if( bDebug ){ + utf8_printf(p->out, "%s\n", zSql); + }else{ + shell_exec(p->db, zSql, shell_callback, p, 0); + } + sqlite3_free(zSql); + }else + if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ From 3ee83efec1a5d12d1339b55f1e028a8db2641047 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 17:56:54 +0000 Subject: [PATCH 224/292] In the ".sha3sum" command, if there is a LIKE pattern, show the hashes for each table separately. Without a LIKE pattern, show a single hash over the entire database. FossilOrigin-Name: 30f878832820ce7ccc4627c4f0f98fbe82f8b0f6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 37 +++++++++++++++++++++++++++++++++---- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 97d4af1fb1..ab4ba0a547 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".sha3sum"\scommand\sto\sthe\sCLI\s-\sused\sto\scompute\sa\scryptographic\shash\nof\sthe\scontent\sof\sa\sdatabase\sfile\sor\sof\sindividual\stables\swith\sin\nthat\sfile. -D 2017-03-08T16:10:34.827 +C In\sthe\s".sha3sum"\scommand,\sif\sthere\sis\sa\sLIKE\spattern,\sshow\sthe\shashes\sfor\neach\stable\sseparately.\s\sWithout\sa\sLIKE\spattern,\sshow\sa\ssingle\shash\sover\sthe\nentire\sdatabase. +D 2017-03-08T17:56:54.566 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 8cebab1fdfb7427216c63a46d9116d31ef06dd55 +F src/shell.c bf10f16352e930e37d26813b5efbf57e89680922 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54ef7abd7f5b16f4b29c9519d283e142c9340fbf -R 1784a0b42056ce37f7cf91782b133804 +P fc663799075a22b0a61a6a114116bb2d1b96d4ab +R ce49cf9f2aa65eda762d2f7f5e9b2cf0 U drh -Z 89ed86e3a2610bc780c3304d2a434090 +Z a6a2f37fa5651fb024df2d5d55e36e09 diff --git a/manifest.uuid b/manifest.uuid index c49a19c8c2..086a9494b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc663799075a22b0a61a6a114116bb2d1b96d4ab \ No newline at end of file +30f878832820ce7ccc4627c4f0f98fbe82f8b0f6 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 06cf36cccc..2beea32552 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1260,6 +1260,10 @@ static void sha3QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); + if( z==0 ){ + sqlite3_finalize(pStmt); + continue; + } n = (int)strlen(z); hash_step_vformat(&cx,"S%d:",n); SHA3Update(&cx,(unsigned char*)z,n); @@ -5701,10 +5705,13 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zLike = 0; /* Which table to checksum. 0 means everything */ int i; /* Loop counter */ int bSchema = 0; /* Also hash the schema */ + int bSeparate = 0; /* Hash each table separately */ int iSize = 224; /* Hash algorithm to use */ int bDebug = 0; /* Only show the query that would have run */ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ + char *zSep; /* Separator */ + ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ for(i=1; idb, zSql, -1, &pStmt, 0); initText(&sQuery); + initText(&sSql); + appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); + zSep = "VALUES("; while( SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zTab = (const char*)sqlite3_column_text(pStmt,0); if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; @@ -5774,11 +5785,29 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sQuery, zTab, 0); appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); } + appendText(&sSql, zSep, 0); + appendText(&sSql, sQuery.z, '\''); + sQuery.n = 0; + appendText(&sSql, ",", 0); + appendText(&sSql, zTab, '\''); + zSep = "),("; } sqlite3_finalize(pStmt); - zSql = sqlite3_mprintf("SELECT lower(hex(sha3_query(%Q,%d))) AS hash;", - sQuery.z, iSize); + if( bSeparate ){ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" + " FROM [sha3sum$query]", + sSql.z, iSize); + }else{ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" + " FROM [sha3sum$query]", + sSql.z, iSize); + } freeText(&sQuery); + freeText(&sSql); if( bDebug ){ utf8_printf(p->out, "%s\n", zSql); }else{ From f80d4ff59f114f58a49d56406875a8c95f900e54 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 18:06:20 +0000 Subject: [PATCH 225/292] Make sure the database connection is open prior to running the ".sha3sum" command. FossilOrigin-Name: 2ea300fb8f7c497f3f092dc91f4305d8431c27d9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ab4ba0a547..4b1051a011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".sha3sum"\scommand,\sif\sthere\sis\sa\sLIKE\spattern,\sshow\sthe\shashes\sfor\neach\stable\sseparately.\s\sWithout\sa\sLIKE\spattern,\sshow\sa\ssingle\shash\sover\sthe\nentire\sdatabase. -D 2017-03-08T17:56:54.566 +C Make\ssure\sthe\sdatabase\sconnection\sis\sopen\sprior\sto\srunning\sthe\s".sha3sum"\ncommand. +D 2017-03-08T18:06:20.965 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c bf10f16352e930e37d26813b5efbf57e89680922 +F src/shell.c 1160c054a483d15213ceb70fdbc383b07eebdf78 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc663799075a22b0a61a6a114116bb2d1b96d4ab -R ce49cf9f2aa65eda762d2f7f5e9b2cf0 +P 30f878832820ce7ccc4627c4f0f98fbe82f8b0f6 +R 15b9925a544b33052292bf7b7cba23e6 U drh -Z a6a2f37fa5651fb024df2d5d55e36e09 +Z d079eee20e78bccc69196e077d629480 diff --git a/manifest.uuid b/manifest.uuid index 086a9494b1..c9f050e5a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30f878832820ce7ccc4627c4f0f98fbe82f8b0f6 \ No newline at end of file +2ea300fb8f7c497f3f092dc91f4305d8431c27d9 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2beea32552..f4d10ef767 100644 --- a/src/shell.c +++ b/src/shell.c @@ -5713,6 +5713,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zSep; /* Separator */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ + open_db(p, 0); for(i=1; i Date: Thu, 9 Mar 2017 13:50:49 +0000 Subject: [PATCH 226/292] Begin moving separate boolean variables in the ShellState object of the CLI into the shellFlgs bitmask. FossilOrigin-Name: 50eec5d9aa38fab1a85d788356ffdaf6c35d9ece --- manifest | 12 +++++----- manifest.uuid | 2 +- src/shell.c | 63 +++++++++++++++++++++++++++++++++------------------ 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 4b1051a011..c8aad6cfb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sdatabase\sconnection\sis\sopen\sprior\sto\srunning\sthe\s".sha3sum"\ncommand. -D 2017-03-08T18:06:20.965 +C Begin\smoving\sseparate\sboolean\svariables\sin\sthe\sShellState\sobject\sof\sthe\sCLI\ninto\sthe\sshellFlgs\sbitmask. +D 2017-03-09T13:50:49.349 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 1160c054a483d15213ceb70fdbc383b07eebdf78 +F src/shell.c 397e51c3eeb3a9dc21667a0a384eb14403cc5eea F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 30f878832820ce7ccc4627c4f0f98fbe82f8b0f6 -R 15b9925a544b33052292bf7b7cba23e6 +P 2ea300fb8f7c497f3f092dc91f4305d8431c27d9 +R 193fb708e3eeb9be14d6692d9fad8ed4 U drh -Z d079eee20e78bccc69196e077d629480 +Z 6ea8a21fbd5a369fc4273386c415b1f4 diff --git a/manifest.uuid b/manifest.uuid index c9f050e5a9..2f86d48c03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ea300fb8f7c497f3f092dc91f4305d8431c27d9 \ No newline at end of file +50eec5d9aa38fab1a85d788356ffdaf6c35d9ece \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index f4d10ef767..1de72f8997 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1361,15 +1361,11 @@ struct SavedModeInfo { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ - int echoOn; /* True to echo input commands */ int autoExplain; /* Automatically turn on .explain mode */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int statsOn; /* True to display memory stats before each finalize */ int scanstatsOn; /* True to display scan stats before each finalize */ - int countChanges; /* True to display change counts */ - int backslashOn; /* Resolve C-style \x escapes in SQL input text */ int outCount; /* Revert to stdout when reaching zero */ - int preserveRowid; /* Preserver ROWID values on a ".dump" command */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ @@ -1407,9 +1403,20 @@ struct ShellState { /* ** These are the allowed shellFlgs values */ -#define SHFLG_Scratch 0x00001 /* The --scratch option is used */ -#define SHFLG_Pagecache 0x00002 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00004 /* Lookaside memory is used */ +#define SHFLG_Scratch 0x00000001 /* The --scratch option is used */ +#define SHFLG_Pagecache 0x00000002 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000004 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000008 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000010 /* .dump preserves rowid values */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ + +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* ** These are the allowed modes. @@ -2639,7 +2646,7 @@ static int shell_exec( } /* echo the sql statement if echo on */ - if( pArg && pArg->echoOn ){ + if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } @@ -2760,7 +2767,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ int nAlloc = 0; int nPK = 0; /* Number of PRIMARY KEY columns seen */ int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ - int preserveRowid = p->preserveRowid; + int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); int rc; zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); @@ -3394,7 +3401,7 @@ static sqlite3_int64 integerValue(const char *zArg){ ** Interpret zArg as either an integer or a boolean value. Return 1 or 0 ** for TRUE and FALSE. Return the integer value if appropriate. */ -static int booleanValue(char *zArg){ +static int booleanValue(const char *zArg){ int i; if( zArg[0]=='0' && zArg[1]=='x' ){ for(i=2; hexDigitValue(zArg[i])>=0; i++){} @@ -3413,6 +3420,17 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Set or clear a shell flag according to a boolean value. +*/ +static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ + if( booleanValue(zArg) ){ + ShellSetFlag(p, mFlag); + }else{ + ShellClearFlag(p, mFlag); + } +} + /* ** Close an output file, assuming it is not stderr or stdout */ @@ -4486,7 +4504,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ - p->countChanges = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ raw_printf(stderr, "Usage: .changes on|off\n"); rc = 1; @@ -4552,13 +4570,13 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0; int i; - p->preserveRowid = 0; + ShellClearFlag(p, SHFLG_PreserveRowid); for(i=1; ipreserveRowid = 1; + ShellSetFlag(p, SHFLG_PreserveRowid); }else { raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]); @@ -4625,7 +4643,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - p->echoOn = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ raw_printf(stderr, "Usage: .echo on|off\n"); rc = 1; @@ -5845,7 +5863,8 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - utf8_printf(p->out, "%12.12s: %s\n","echo", azBool[p->echoOn!=0]); + utf8_printf(p->out, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); utf8_printf(p->out, "%12.12s: %s\n","explain", p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); @@ -6410,7 +6429,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ char *zErrMsg = 0; open_db(p, 0); - if( p->backslashOn ) resolve_backslashes(zSql); + if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); BEGIN_TIMER; rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg); END_TIMER; @@ -6430,7 +6449,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); } return 1; - }else if( p->countChanges ){ + }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ raw_printf(p->out, "changes: %3d total_changes: %d\n", sqlite3_changes(p->db), sqlite3_total_changes(p->db)); } @@ -6473,11 +6492,11 @@ static int process_input(ShellState *p, FILE *in){ } lineno++; if( nSql==0 && _all_whitespace(zLine) ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); continue; } if( zLine && zLine[0]=='.' && nSql==0 ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); rc = do_meta_command(zLine, p); if( rc==2 ){ /* exit requested */ break; @@ -6520,7 +6539,7 @@ static int process_input(ShellState *p, FILE *in){ p->outCount = 0; } }else if( nSql && _all_whitespace(zSql) ){ - if( p->echoOn ) printf("%s\n", zSql); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); nSql = 0; } } @@ -6989,7 +7008,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( strcmp(z,"-noheader")==0 ){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ - data.echoOn = 1; + ShellSetFlag(&data, SHFLG_Echo); }else if( strcmp(z,"-eqp")==0 ){ data.autoEQP = 1; }else if( strcmp(z,"-eqpfull")==0 ){ @@ -7004,7 +7023,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** prior to sending the SQL into SQLite. Useful for injecting ** crazy bytes in the middle of SQL statements for testing and debugging. */ - data.backslashOn = 1; + ShellSetFlag(&data, SHFLG_Backslash); }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ From f8563c00b277ef39608148644660d697b7707f53 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Mar 2017 18:13:52 +0000 Subject: [PATCH 227/292] Fix the ".dump" command to correctly extract tail data from corrupt WITHOUT ROWID tables. FossilOrigin-Name: 6c627e50622d8bcd25ec7d5503f3fafd725673a8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c8aad6cfb1..bfcca0a0ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\smoving\sseparate\sboolean\svariables\sin\sthe\sShellState\sobject\sof\sthe\sCLI\ninto\sthe\sshellFlgs\sbitmask. -D 2017-03-09T13:50:49.349 +C Fix\sthe\s".dump"\scommand\sto\scorrectly\sextract\stail\sdata\sfrom\scorrupt\nWITHOUT\sROWID\stables. +D 2017-03-09T18:13:52.322 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 397e51c3eeb3a9dc21667a0a384eb14403cc5eea +F src/shell.c f4a7169ddfff73ba1ab2f06a4e97bd6d569cb984 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2ea300fb8f7c497f3f092dc91f4305d8431c27d9 -R 193fb708e3eeb9be14d6692d9fad8ed4 +P 50eec5d9aa38fab1a85d788356ffdaf6c35d9ece +R a2e2297f06f9d9483ad6626d70cfa2bc U drh -Z 6ea8a21fbd5a369fc4273386c415b1f4 +Z 704977d7ee0085e38c9f718cc949a370 diff --git a/manifest.uuid b/manifest.uuid index 2f86d48c03..53f69764d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50eec5d9aa38fab1a85d788356ffdaf6c35d9ece \ No newline at end of file +6c627e50622d8bcd25ec7d5503f3fafd725673a8 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1de72f8997..adc57481b8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2850,6 +2850,23 @@ static char **tableColumnList(ShellState *p, const char *zTab){ return azCol; } +/* +** Toggle the reverse_unordered_selects setting. +*/ +static void toggleSelectOrder(sqlite3 *db){ + sqlite3_stmt *pStmt = 0; + int iSetting = 0; + char zStmt[100]; + sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iSetting = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + sqlite3_snprintf(sizeof(zStmt), zStmt, + "PRAGMA reverse_unordered_selects(%d)", !iSetting); + sqlite3_exec(db, zStmt, 0, 0, 0); +} + /* ** This is a different callback routine used for dumping the database. ** Each row received by this callback consists of a table name, @@ -2946,6 +2963,12 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ p->zDestTable = sTable.z; p->mode = p->cMode = MODE_Insert; rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + if( (rc&0xff)==SQLITE_CORRUPT ){ + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + toggleSelectOrder(p->db); + shell_exec(p->db, sSelect.z, shell_callback, p, 0); + toggleSelectOrder(p->db); + } p->zDestTable = savedDestTable; p->mode = savedMode; freeText(&sTable); From fb546afb4dfa625b657ecd48b55cd936be617f4f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Mar 2017 22:00:33 +0000 Subject: [PATCH 228/292] Remove the test/dbselftest.c program. In its place, add the ".selftest" command to the CLI. The new CLI version is .selftest is slightly different in that it uses SHA3 hashing instead of SHA1, so the new is subtly incompatible with the old. FossilOrigin-Name: f4fcd46f08ba59d2a3e772cad98383129f648386 --- Makefile.in | 5 - main.mk | 5 - manifest | 17 +- manifest.uuid | 2 +- src/shell.c | 161 ++++++++++ test/dbselftest.c | 786 ---------------------------------------------- 6 files changed, 170 insertions(+), 806 deletions(-) delete mode 100644 test/dbselftest.c diff --git a/Makefile.in b/Makefile.in index 740b49c08f..d5fa831e6b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1185,11 +1185,6 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) -DBSELFTEST_OPT += -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 - -dbselftest$(TEXE): $(TOP)/test/dbselftest.c sqlite3.c - $(LTLINK) $(DBSELFTEST_OPT) -o $@ $(TOP)/test/dbselftest.c sqlite3.c $(TLIBS) - rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) diff --git a/main.mk b/main.mk index 85ed76a143..57f09ff76c 100644 --- a/main.mk +++ b/main.mk @@ -479,8 +479,6 @@ FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ -DBSELFTEST_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DBSELFTEST_OPT += -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here @@ -902,9 +900,6 @@ speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c $(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) -dbselftest$(EXE): $(TOP)/test/dbselftest.c sqlite3.c - $(TCCX) -I. $(DBSELFTEST_OPT) -o dbselftest$(EXE) $(TOP)/test/dbselftest.c sqlite3.c $(THREADLIB) - rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ $(THREADLIB) diff --git a/manifest b/manifest index bfcca0a0ba..3b64809139 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Fix\sthe\s".dump"\scommand\sto\scorrectly\sextract\stail\sdata\sfrom\scorrupt\nWITHOUT\sROWID\stables. -D 2017-03-09T18:13:52.322 -F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 +C Remove\sthe\stest/dbselftest.c\sprogram.\s\sIn\sits\splace,\sadd\sthe\s".selftest"\ncommand\sto\sthe\sCLI.\s\sThe\snew\sCLI\sversion\sis\s.selftest\sis\sslightly\sdifferent\nin\sthat\sit\suses\sSHA3\shashing\sinstead\sof\sSHA1,\sso\sthe\snew\sis\ssubtly\s\nincompatible\swith\sthe\sold. +D 2017-03-09T22:00:33.842 +F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -323,7 +323,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 08e0b151abb46353c677ac4974b50c6077f85eee +F main.mk 98f9e673437e28b17f86d07d0749021bb140c152 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c f4a7169ddfff73ba1ab2f06a4e97bd6d569cb984 +F src/shell.c 2009654e24924b51a54afb925b49034f1c5460d4 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -643,7 +643,6 @@ F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbselftest.c b2e6cfac59066dbcb7334b66304bb15a5508dd42 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1564,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50eec5d9aa38fab1a85d788356ffdaf6c35d9ece -R a2e2297f06f9d9483ad6626d70cfa2bc +P 6c627e50622d8bcd25ec7d5503f3fafd725673a8 +R de6856f188fdc076241cbc67ee174b05 U drh -Z 704977d7ee0085e38c9f718cc949a370 +Z 85b6274077eac0b5500dd53648c9e848 diff --git a/manifest.uuid b/manifest.uuid index 53f69764d4..e2ef7d5abf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c627e50622d8bcd25ec7d5503f3fafd725673a8 \ No newline at end of file +f4fcd46f08ba59d2a3e772cad98383129f648386 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index adc57481b8..17048adb3d 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2054,6 +2054,49 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ return shell_callback(pArg, nArg, azArg, azCol, NULL); } +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "CREATE TABLE selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "INSERT INTO selftest(op,cmd,ans)\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_master\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO selftest(op,cmd,ans)\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + ,0,0,0); +} + /* ** Set the destination table field of the ShellState structure to @@ -5727,6 +5770,124 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif + if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + int bIsInit = 0; /* True to initialize the SELFTEST table */ + int bVerbose = 0; /* Verbose output */ + int bSelftestExists; /* True if SELFTEST already exists */ + char **azTest = 0; /* Content of the SELFTEST table */ + int nRow = 0; /* Number of rows in the SELFTEST table */ + int nCol = 4; /* Number of columns in the SELFTEST table */ + int i; /* Loop counter */ + int nTest = 0; /* Number of tests runs */ + int nErr = 0; /* Number of errors seen */ + ShellText str; /* Answer for a query */ + static char *azDefaultTest[] = { + 0, 0, 0, 0, + "0", "memo", "Missing SELFTEST table - default checks only", "", + "1", "run", "PRAGMA integrity_check", "ok" + }; + static const int nDefaultRow = 2; + + open_db(p,0); + for(i=1; idb,"main","selftest",0,0,0,0,0,0) + != SQLITE_OK ){ + bSelftestExists = 0; + }else{ + bSelftestExists = 1; + } + if( bIsInit ){ + if( bSelftestExists ){ + raw_printf(stderr, "The selftest table already exists\n"); + rc = 1; + goto meta_command_exit; + } + createSelftestTable(p); + bSelftestExists = 1; + } + if( bSelftestExists ){ + rc = sqlite3_get_table(p->db, + "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno", + &azTest, &nRow, &nCol, 0); + if( rc ){ + raw_printf(stderr, "Error querying the selftest table\n"); + rc = 1; + sqlite3_free_table(azTest); + goto meta_command_exit; + }else if( nRow==0 ){ + sqlite3_free_table(azTest); + azTest = azDefaultTest; + nRow = nDefaultRow; + } + }else{ + azTest = azDefaultTest; + nRow = nDefaultRow; + } + initText(&str); + appendText(&str, "x", 0); + for(i=1; i<=nRow; i++){ + int tno = atoi(azTest[i*nCol]); + const char *zOp = azTest[i*nCol+1]; + const char *zSql = azTest[i*nCol+2]; + const char *zAns = azTest[i*nCol+3]; + + if( bVerbose>0 ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + utf8_printf(p->out, "%s\n", zSql); + }else + if( strcmp(zOp,"run")==0 ){ + char *zErrMsg = 0; + str.n = 0; + str.z[0] = 0; + rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); + nTest++; + if( bVerbose ){ + utf8_printf(p->out, "Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + } + }else + { + utf8_printf(stderr, + "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + rc = 1; + break; + } + } + freeText(&str); + if( azTest!=azDefaultTest ) sqlite3_free_table(azTest); + utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + }else + if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); diff --git a/test/dbselftest.c b/test/dbselftest.c deleted file mode 100644 index 3a238bce16..0000000000 --- a/test/dbselftest.c +++ /dev/null @@ -1,786 +0,0 @@ -/* -** 2017-02-07 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This program implements an SQLite database self-verification utility. -** Usage: -** -** dbselftest DATABASE ... -** -** This program reads the "selftest" table in DATABASE, in rowid order, -** and runs each of the tests described there, reporting results at the -** end. -** -** The intent of this program is to have a set of test database files that -** can be run using future versions of SQLite in order to verify that -** legacy database files continue to be readable. In other words, the -** intent is to confirm that there have been no breaking changes in the -** file format. The program can also be used to verify that database files -** are fully compatible between different architectures. -** -** The selftest table looks like this: -** -** CREATE TABLE selftest ( -** id INTEGER PRIMARY KEY, -- Run tests in ascending order -** op TEXT, -- "test", "regexp", "print", etc. -** cmdtxt TEXT, -- Usually the SQL to be run -** expected TEXT -- Expected results -** ); -** -*/ -#include -#include -#include -#include -#include -#include "sqlite3.h" - -static const char zHelp[] = - "Usage: dbselftest [OPTIONS] DBFILE ...\n" - "\n" - " --init Create the selftest table\n" - " -q Suppress most output. Errors only\n" - " -v Show extra output\n" -; - - -/****************************************************************************** -** The following code from ext/misc/sha1.c -** -** Context for the SHA1 hash -*/ -typedef struct SHA1Context SHA1Context; -struct SHA1Context { - unsigned int state[5]; - unsigned int count[2]; - unsigned char buffer[64]; -}; - - -#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) -/* - * GCC by itself only generates left rotates. Use right rotates if - * possible to be kinder to dinky implementations with iterative rotate - * instructions. - */ -#define SHA_ROT(op, x, k) \ - ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) -#define rol(x,k) SHA_ROT("roll", x, k) -#define ror(x,k) SHA_ROT("rorl", x, k) - -#else -/* Generic C equivalent */ -#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) -#define rol(x,k) SHA_ROT(x,k,32-(k)) -#define ror(x,k) SHA_ROT(x,32-(k),k) -#endif - - -#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ - |(rol(block[i],8)&0x00FF00FF)) -#define blk0be(i) block[i] -#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ - ^block[(i+2)&15]^block[i&15],1)) - -/* - * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 - * - * Rl0() for little-endian and Rb0() for big-endian. Endianness is - * determined at run-time. - */ -#define Rl0(v,w,x,y,z,i) \ - z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); -#define Rb0(v,w,x,y,z,i) \ - z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); -#define R1(v,w,x,y,z,i) \ - z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); -#define R2(v,w,x,y,z,i) \ - z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); -#define R3(v,w,x,y,z,i) \ - z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); -#define R4(v,w,x,y,z,i) \ - z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); - -/* - * Hash a single 512-bit block. This is the core of the algorithm. - */ -void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ - unsigned int qq[5]; /* a, b, c, d, e; */ - static int one = 1; - unsigned int block[16]; - memcpy(block, buffer, 64); - memcpy(qq,state,5*sizeof(unsigned int)); - -#define a qq[0] -#define b qq[1] -#define c qq[2] -#define d qq[3] -#define e qq[4] - - /* Copy p->state[] to working vars */ - /* - a = state[0]; - b = state[1]; - c = state[2]; - d = state[3]; - e = state[4]; - */ - - /* 4 rounds of 20 operations each. Loop unrolled. */ - if( 1 == *(unsigned char*)&one ){ - Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); - Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); - Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); - Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); - }else{ - Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); - Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); - Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); - Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); - } - R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); - R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); - R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); - R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); - R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); - R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); - R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); - R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); - R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); - R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); - R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); - R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); - R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); - R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); - R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); - R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); - - /* Add the working vars back into context.state[] */ - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; - -#undef a -#undef b -#undef c -#undef d -#undef e -} - - -/* Initialize a SHA1 context */ -static void hash_init(SHA1Context *p){ - /* SHA1 initialization constants */ - p->state[0] = 0x67452301; - p->state[1] = 0xEFCDAB89; - p->state[2] = 0x98BADCFE; - p->state[3] = 0x10325476; - p->state[4] = 0xC3D2E1F0; - p->count[0] = p->count[1] = 0; -} - -/* Add new content to the SHA1 hash */ -static void hash_step( - SHA1Context *p, /* Add content to this context */ - const unsigned char *data, /* Data to be added */ - unsigned int len /* Number of bytes in data */ -){ - unsigned int i, j; - - j = p->count[0]; - if( (p->count[0] += len << 3) < j ){ - p->count[1] += (len>>29)+1; - } - j = (j >> 3) & 63; - if( (j + len) > 63 ){ - (void)memcpy(&p->buffer[j], data, (i = 64-j)); - SHA1Transform(p->state, p->buffer); - for(; i + 63 < len; i += 64){ - SHA1Transform(p->state, &data[i]); - } - j = 0; - }else{ - i = 0; - } - (void)memcpy(&p->buffer[j], &data[i], len - i); -} - -/* Compute a string using sqlite3_vsnprintf() and hash it */ -static void hash_step_vformat( - SHA1Context *p, /* Add content to this context */ - const char *zFormat, - ... -){ - va_list ap; - int n; - char zBuf[50]; - va_start(ap, zFormat); - sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); - va_end(ap); - n = (int)strlen(zBuf); - hash_step(p, (unsigned char*)zBuf, n); -} - - -/* Add padding and compute the message digest. Render the -** message digest as lower-case hexadecimal and put it into -** zOut[]. zOut[] must be at least 41 bytes long. */ -static void hash_finish( - SHA1Context *p, /* The SHA1 context to finish and render */ - char *zOut /* Store hexadecimal hash here */ -){ - unsigned int i; - unsigned char finalcount[8]; - unsigned char digest[20]; - static const char zEncode[] = "0123456789abcdef"; - - for (i = 0; i < 8; i++){ - finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] - >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ - } - hash_step(p, (const unsigned char *)"\200", 1); - while ((p->count[0] & 504) != 448){ - hash_step(p, (const unsigned char *)"\0", 1); - } - hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ - for (i = 0; i < 20; i++){ - digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); - } - for(i=0; i<20; i++){ - zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; - zOut[i*2+1] = zEncode[digest[i] & 0xf]; - } - zOut[i*2]= 0; -} - -/* -** Implementation of the sha1(X) function. -** -** Return a lower-case hexadecimal rendering of the SHA1 hash of the -** argument X. If X is a BLOB, it is hashed as is. For all other -** types of input, X is converted into a UTF-8 string and the string -** is hash without the trailing 0x00 terminator. The hash of a NULL -** value is NULL. -*/ -static void sha1Func( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - SHA1Context cx; - int eType = sqlite3_value_type(argv[0]); - int nByte = sqlite3_value_bytes(argv[0]); - char zOut[44]; - - assert( argc==1 ); - if( eType==SQLITE_NULL ) return; - hash_init(&cx); - if( eType==SQLITE_BLOB ){ - hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); - }else{ - hash_step(&cx, sqlite3_value_text(argv[0]), nByte); - } - hash_finish(&cx, zOut); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); -} - -/* -** Run a prepared statement and compute the SHA1 hash on the -** result rows. -*/ -static void sha1RunStatement(SHA1Context *pCtx, sqlite3_stmt *pStmt){ - int nCol = sqlite3_column_count(pStmt); - const char *z = sqlite3_sql(pStmt); - int n = (int)strlen(z); - - hash_step_vformat(pCtx,"S%d:",n); - hash_step(pCtx,(unsigned char*)z,n); - - /* Compute a hash over the result of the query */ - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - int i; - hash_step(pCtx,(const unsigned char*)"R",1); - for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - hash_step(pCtx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - hash_step(pCtx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(pCtx,"T%d:",n2); - hash_step(pCtx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(pCtx,"B%d:",n2); - hash_step(pCtx, z2, n2); - break; - } - } - } - } -} - -/* -** Run one or more statements of SQL. Compute a SHA1 hash of the output. -*/ -static int sha1Exec( - sqlite3 *db, /* Run against this database connection */ - const char *zSql, /* The SQL to be run */ - char *zOut /* Store the SHA1 hash as hexadecimal in this buffer */ -){ - sqlite3_stmt *pStmt = 0; /* A prepared statement */ - int rc; /* Result of an API call */ - SHA1Context cx; /* The SHA1 hash context */ - - hash_init(&cx); - while( zSql[0] ){ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc ){ - sqlite3_finalize(pStmt); - return rc; - } - sha1RunStatement(&cx, pStmt); - sqlite3_finalize(pStmt); - } - hash_finish(&cx, zOut); - return SQLITE_OK; -} - -/* -** Implementation of the sha1_query(SQL) function. -** -** This function compiles and runs the SQL statement(s) given in the -** argument. The results are hashed using SHA1 and that hash is returned. -** -** The original SQL text is included as part of the hash. -** -** The hash is not just a concatenation of the outputs. Each query -** is delimited and each row and value within the query is delimited, -** with all values being marked with their datatypes. -*/ -static void sha1QueryFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - const char *zSql = (const char*)sqlite3_value_text(argv[0]); - sqlite3_stmt *pStmt = 0; - int rc; - SHA1Context cx; - char zOut[44]; - - assert( argc==1 ); - if( zSql==0 ) return; - hash_init(&cx); - while( zSql[0] ){ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc ){ - char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", - zSql, sqlite3_errmsg(db)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - if( !sqlite3_stmt_readonly(pStmt) ){ - char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - sha1RunStatement(&cx, pStmt); - sqlite3_finalize(pStmt); - } - hash_finish(&cx, zOut); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); -} -/* End of ext/misc/sha1.c -******************************************************************************/ - -/* How much output to display */ -#define VOLUME_MIN 0 -#define VOLUME_OFF 0 -#define VOLUME_ERROR_ONLY 1 -#define VOLUME_LOW 2 -#define VOLUME_ECHO 3 -#define VOLUME_VERBOSE 4 -#define VOLUME_MAX 4 - -/* A string accumulator -*/ -typedef struct Str { - char *z; /* Accumulated text */ - int n; /* Bytes of z[] used so far */ - int nAlloc; /* Bytes allocated for z[] */ -} Str; - -/* Append text to the Str object -*/ -static void strAppend(Str *p, const char *z){ - int n = (int)strlen(z); - if( p->n+n >= p->nAlloc ){ - p->nAlloc += p->n+n + 100; - p->z = sqlite3_realloc(p->z, p->nAlloc); - if( z==0 ){ - printf("Could not allocate %d bytes\n", p->nAlloc); - exit(1); - } - } - memcpy(p->z+p->n, z, n+1); - p->n += n; -} - -/* This is an sqlite3_exec() callback that will capture all -** output in a Str. -** -** Columns are separated by ",". Rows are separated by "|". -*/ -static int execCallback(void *pStr, int argc, char **argv, char **colv){ - int i; - Str *p = (Str*)pStr; - if( p->n ) strAppend(p, "|"); - for(i=0; i0 ) strAppend(p, ","); - strAppend(p, z); - } - return 0; -} - -/* -** Run an SQL statement constructing using sqlite3_vmprintf(). -** Return the number of errors. -*/ -static int runSql(sqlite3 *db, const char *zFormat, ...){ - char *zSql; - char *zErr = 0; - int rc; - int nErr = 0; - va_list ap; - - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - printf("Out of memory\n"); - exit(1); - } - rc = sqlite3_exec(db, zSql, 0, 0, &zErr); - if( rc || zErr ){ - printf("SQL error in [%s]: code=%d: %s\n", zSql, rc, zErr); - nErr++; - } - sqlite3_free(zSql); - return nErr; -} - -/* -** Generate a prepared statement using a formatted string. -*/ -static sqlite3_stmt *prepareSql(sqlite3 *db, const char *zFormat, ...){ - char *zSql; - int rc; - sqlite3_stmt *pStmt = 0; - va_list ap; - - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - printf("Out of memory\n"); - exit(1); - } - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc ){ - printf("SQL error in [%s]: code=%d: %s\n", zSql, rc, sqlite3_errmsg(db)); - sqlite3_finalize(pStmt); - pStmt = 0; - } - sqlite3_free(zSql); - return pStmt; -} - -/* -** Construct the standard selftest configuration for the database. -*/ -static int buildSelftestTable(sqlite3 *db){ - int rc; - sqlite3_stmt *pStmt; - int tno = 110; - char *zSql; - char zHash[50]; - - rc = runSql(db, - "CREATE TABLE IF NOT EXISTS selftest(\n" - " tno INTEGER PRIMARY KEY, -- test number\n" - " op TEXT, -- what kind of test\n" - " sql TEXT, -- SQL text for the test\n" - " ans TEXT -- expected answer\n" - ");" - "INSERT INTO selftest" - " VALUES(100,'memo','Hashes generated using --init',NULL);" - ); - if( rc ) return 1; - tno = 110; - zSql = "SELECT type,name,tbl_name,sql FROM sqlite_master ORDER BY name"; - sha1Exec(db, zSql, zHash); - rc = runSql(db, - "INSERT INTO selftest(tno,op,sql,ans)" - " VALUES(%d,'sha1',%Q,%Q)", tno, zSql, zHash); - tno += 10; - pStmt = prepareSql(db, - "SELECT lower(name) FROM sqlite_master" - " WHERE type='table' AND sql NOT GLOB 'CREATE VIRTUAL*'" - " AND name<>'selftest'" - " ORDER BY 1"); - if( pStmt==0 ) return 1; - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - zSql = sqlite3_mprintf("SELECT * FROM \"%w\" NOT INDEXED", - sqlite3_column_text(pStmt, 0)); - if( zSql==0 ){ - printf("Of of memory\n"); - exit(1); - } - sha1Exec(db, zSql, zHash); - rc = runSql(db, - "INSERT INTO selftest(tno,op,sql,ans)" - " VALUES(%d,'sha1',%Q,%Q)", tno, zSql, zHash); - tno += 10; - sqlite3_free(zSql); - if( rc ) break; - } - sqlite3_finalize(pStmt); - if( rc ) return 1; - rc = runSql(db, - "INSERT INTO selftest(tno,op,sql,ans)" - " VALUES(%d,'run','PRAGMA integrity_check','ok');", tno); - if( rc ) return 1; - return rc; -} - -/* -** Return true if the named table exists -*/ -static int tableExists(sqlite3 *db, const char *zTab){ - return sqlite3_table_column_metadata(db, "main", zTab, 0, 0, 0, 0, 0, 0) - == SQLITE_OK; -} - -/* -** Default selftest table content, for use when there is no selftest table -*/ -static char *azDefaultTest[] = { - 0, 0, 0, 0, - "0", "memo", "Missing SELFTEST table - default checks only", "", - "1", "run", "PRAGMA integrity_check", "ok" -}; - -int main(int argc, char **argv){ - int eVolume = VOLUME_LOW; /* How much output to display */ - const char **azDb = 0; /* Name of the database file */ - int nDb = 0; /* Number of database files to check */ - int doInit = 0; /* True if --init is present */ - sqlite3 *db = 0; /* Open database connection */ - int rc; /* Return code from API calls */ - char *zErrMsg = 0; /* An error message return */ - char **azTest; /* Content of the selftest table */ - int nRow = 0, nCol = 0; /* Rows and columns in azTest[] */ - int i; /* Loop counter */ - int nErr = 0; /* Number of errors */ - int iDb; /* Loop counter for databases */ - Str str; /* Result accumulator */ - int nTest = 0; /* Number of tests run */ - - for(i=1; iVOLUME_MIN) eVolume--; - }else - if( strcmp(z, "-v")==0 ){ - if( eVolume=VOLUME_LOW ){ - printf("SQLite %s\n", sqlite3_sourceid()); - } - memset(&str, 0, sizeof(str)); - strAppend(&str, "\n"); - for(iDb=0; iDb=VOLUME_ECHO ){ - char *zQuote = sqlite3_mprintf("%q", zSql); - printf("%d: %s %s\n", tno, zOp, zSql); - sqlite3_free(zQuote); - } - if( strcmp(zOp,"memo")==0 ){ - if( eVolume>=VOLUME_LOW ){ - printf("%s: %s\n", azDb[iDb], zSql); - } - }else - if( strcmp(zOp,"sha1")==0 ){ - char zOut[44]; - rc = sha1Exec(db, zSql, zOut); - nTest++; - if( eVolume>=VOLUME_VERBOSE ){ - printf("Result: %s\n", zOut); - } - if( rc ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: error-code-%d: %s\n", tno, rc, sqlite3_errmsg(db)); - } - }else if( strcmp(zAns,zOut)!=0 ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: Expected: [%s]\n", tno, zAns); - printf("%d: Got: [%s]\n", tno, zOut); - } - } - }else - if( strcmp(zOp,"run")==0 ){ - str.n = 0; - str.z[0] = 0; - zErrMsg = 0; - rc = sqlite3_exec(db, zSql, execCallback, &str, &zErrMsg); - nTest++; - if( eVolume>=VOLUME_VERBOSE ){ - printf("Result: %s\n", str.z); - } - if( rc || zErrMsg ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); - } - sqlite3_free(zErrMsg); - }else if( strcmp(zAns,str.z)!=0 ){ - nErr++; - if( eVolume>=VOLUME_ERROR_ONLY ){ - printf("%d: Expected: [%s]\n", tno, zAns); - printf("%d: Got: [%s]\n", tno, str.z); - } - } - }else - { - printf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); - return 1; - } - } - if( azTest!=azDefaultTest ) sqlite3_free_table(azTest); - } - if( eVolume>=VOLUME_LOW || (nErr>0 && eVolume>=VOLUME_ERROR_ONLY) ){ - printf("%d errors out of %d tests on %d databases\n", nErr, nTest, nDb); - } - return nErr; -} From f157d10f9f557e4117d3267eba89d817141feead Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Mar 2017 01:05:38 +0000 Subject: [PATCH 229/292] Improvements to ".selftest --init". Tests are number in increments of 10 starting with 100. The tests are generated inside a SAVEPOINT. Errors are reported during test generation. Tests can be appended to existing tests. Add a test case to verify the schema. FossilOrigin-Name: b044b152aac2ec606750940ea816ad4a4aef8eb6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 33 ++++++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 3b64809139..5ebdb5663f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\stest/dbselftest.c\sprogram.\s\sIn\sits\splace,\sadd\sthe\s".selftest"\ncommand\sto\sthe\sCLI.\s\sThe\snew\sCLI\sversion\sis\s.selftest\sis\sslightly\sdifferent\nin\sthat\sit\suses\sSHA3\shashing\sinstead\sof\sSHA1,\sso\sthe\snew\sis\ssubtly\s\nincompatible\swith\sthe\sold. -D 2017-03-09T22:00:33.842 +C Improvements\sto\s".selftest\s--init".\s\sTests\sare\snumber\sin\sincrements\sof\s10\nstarting\swith\s100.\s\sThe\stests\sare\sgenerated\sinside\sa\sSAVEPOINT.\s\sErrors\sare\nreported\sduring\stest\sgeneration.\s\sTests\scan\sbe\sappended\sto\sexisting\stests.\nAdd\sa\stest\scase\sto\sverify\sthe\sschema. +D 2017-03-10T01:05:38.013 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 2009654e24924b51a54afb925b49034f1c5460d4 +F src/shell.c 0435e7ea18865e8bd435c9fbe018e1a8431964a7 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6c627e50622d8bcd25ec7d5503f3fafd725673a8 -R de6856f188fdc076241cbc67ee174b05 +P f4fcd46f08ba59d2a3e772cad98383129f648386 +R 2a153e04b79acc8e7981c04d4b205835 U drh -Z 85b6274077eac0b5500dd53648c9e848 +Z c20519a5ebf6e7586b129dc9c77c4f4b diff --git a/manifest.uuid b/manifest.uuid index e2ef7d5abf..d8526f73db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4fcd46f08ba59d2a3e772cad98383129f648386 \ No newline at end of file +b044b152aac2ec606750940ea816ad4a4aef8eb6 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 17048adb3d..4912639dbc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2073,14 +2073,26 @@ static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ ** Generate an appropriate SELFTEST table in the main database. */ static void createSelftestTable(ShellState *p){ + char *zErrMsg = 0; sqlite3_exec(p->db, - "CREATE TABLE selftest(\n" + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" " tno INTEGER PRIMARY KEY,\n" /* Test number */ " op TEXT,\n" /* Operator: memo run */ " cmd TEXT,\n" /* Command text */ " ans TEXT\n" /* Desired answer */ ");" - "INSERT INTO selftest(op,cmd,ans)\n" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" " SELECT 'run'," " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" " printf('%w',name) || '\" NOT INDEXED'',224))',\n" @@ -2092,9 +2104,17 @@ static void createSelftestTable(ShellState *p){ " AND coalesce(rootpage,0)>0\n" " )\n" " ORDER BY name;\n" - "INSERT INTO selftest(op,cmd,ans)\n" + "INSERT INTO [_shell$self]\n" " VALUES('run','PRAGMA integrity_check','ok');\n" - ,0,0,0); + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); } @@ -5813,11 +5833,6 @@ static int do_meta_command(char *zLine, ShellState *p){ bSelftestExists = 1; } if( bIsInit ){ - if( bSelftestExists ){ - raw_printf(stderr, "The selftest table already exists\n"); - rc = 1; - goto meta_command_exit; - } createSelftestTable(p); bSelftestExists = 1; } From a2df53bd619520ce4ea8be314bed25d998a13947 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Mar 2017 14:36:10 +0000 Subject: [PATCH 230/292] Enhance the ".stats" dot-command in the CLI to use sqlite3_status64() instead of sqlite3_status(). FossilOrigin-Name: 118f5c0564fef70cbd06fc0d9dbb2baec162cc39 --- manifest | 12 +++---- manifest.uuid | 2 +- src/shell.c | 91 +++++++++++++++++++++++++-------------------------- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index 5ebdb5663f..5c43042028 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\s".selftest\s--init".\s\sTests\sare\snumber\sin\sincrements\sof\s10\nstarting\swith\s100.\s\sThe\stests\sare\sgenerated\sinside\sa\sSAVEPOINT.\s\sErrors\sare\nreported\sduring\stest\sgeneration.\s\sTests\scan\sbe\sappended\sto\sexisting\stests.\nAdd\sa\stest\scase\sto\sverify\sthe\sschema. -D 2017-03-10T01:05:38.013 +C Enhance\sthe\s".stats"\sdot-command\sin\sthe\sCLI\sto\suse\ssqlite3_status64()\sinstead\nof\ssqlite3_status(). +D 2017-03-10T14:36:10.584 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 0435e7ea18865e8bd435c9fbe018e1a8431964a7 +F src/shell.c 4d9be7f0622365029a1f2e0f9628b8e363700ff5 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4fcd46f08ba59d2a3e772cad98383129f648386 -R 2a153e04b79acc8e7981c04d4b205835 +P b044b152aac2ec606750940ea816ad4a4aef8eb6 +R da300103282ee8980f2e41cc2466def0 U drh -Z c20519a5ebf6e7586b129dc9c77c4f4b +Z 6dbad136ee05d7b8bc7f34310ddbaf47 diff --git a/manifest.uuid b/manifest.uuid index d8526f73db..f610b09f28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b044b152aac2ec606750940ea816ad4a4aef8eb6 \ No newline at end of file +118f5c0564fef70cbd06fc0d9dbb2baec162cc39 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 4912639dbc..378729d09b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2259,6 +2259,31 @@ static void displayLinuxIoStats(FILE *out){ } #endif +/* +** Display a single line of status using 64-bit values. +*/ +static void displayStatLine( + ShellState *p, /* The shell context */ + char *zLabel, /* Label for this one line */ + char *zFormat, /* Format for the result */ + int iStatusCtrl, /* Which status to display */ + int bReset /* True to reset the stats */ +){ + sqlite3_int64 iCur = -1; + sqlite3_int64 iHiwtr = -1; + int i, nPercent; + char zLine[200]; + sqlite3_status64(iStatusCtrl, &iCur, &iHiwtr, bReset); + for(i=0, nPercent=0; zFormat[i]; i++){ + if( zFormat[i]=='%' ) nPercent++; + } + if( nPercent>1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + raw_printf(p->out, "%-36s %s\n", zLabel, zLine); +} /* ** Display memory stats. @@ -2272,57 +2297,31 @@ static int display_stats( int iHiwtr; if( pArg && pArg->out ){ - - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Memory Used: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Pages Used: %d (max %d) pages\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); if( pArg->shellFlgs & SHFLG_Scratch ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Allocations Used: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Scratch Allocations Used:", + "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Pcache Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Scratch Allocation: %d bytes\n", - iHiwtr); + displayStatLine(pArg, "Number of Scratch Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); + displayStatLine(pArg, "Largest Scratch Allocation:", + "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif } From e43d6aee45208884330cd23a64d2e38771b8f5e4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Mar 2017 15:55:54 +0000 Subject: [PATCH 231/292] Add the SQLITE_MAX_MEMORY compile-time option that provides a hard upper bound on the amount of memory that SQLite will use, per process. FossilOrigin-Name: 77dfe2abdae88dea81217f352d87e5ba2c822715 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/malloc.c | 12 +++++++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5c43042028..05fc15b13a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s".stats"\sdot-command\sin\sthe\sCLI\sto\suse\ssqlite3_status64()\sinstead\nof\ssqlite3_status(). -D 2017-03-10T14:36:10.584 +C Add\sthe\sSQLITE_MAX_MEMORY\scompile-time\soption\sthat\sprovides\sa\shard\supper\sbound\non\sthe\samount\sof\smemory\sthat\sSQLite\swill\suse,\sper\sprocess. +D 2017-03-10T15:55:54.959 F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -365,7 +365,7 @@ F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 -F src/malloc.c d0a1474236486165bcb349af82e2a6560178bf7b +F src/malloc.c 96f86a14b04fce2cb9559ccb3a2ae4534131094e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1563,7 +1563,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b044b152aac2ec606750940ea816ad4a4aef8eb6 -R da300103282ee8980f2e41cc2466def0 +P 118f5c0564fef70cbd06fc0d9dbb2baec162cc39 +R 32dd2e22d477f93aeae962c3294c4485 +T *branch * max-memory-option +T *sym-max-memory-option * +T -sym-trunk * U drh -Z 6dbad136ee05d7b8bc7f34310ddbaf47 +Z 9eb3e2d252d807f2c1277379e9be3f8a diff --git a/manifest.uuid b/manifest.uuid index f610b09f28..27ef1bc7b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -118f5c0564fef70cbd06fc0d9dbb2baec162cc39 \ No newline at end of file +77dfe2abdae88dea81217f352d87e5ba2c822715 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 0fdc8d73ff..b942f47b28 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -217,7 +217,7 @@ static void sqlite3MallocAlarm(int nByte){ ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. */ -static void mallocWithAlarm(int n, void **pp){ +static void *mallocWithAlarm(int n){ void *p; int nFull; assert( sqlite3_mutex_held(mem0.mutex) ); @@ -230,6 +230,12 @@ static void mallocWithAlarm(int n, void **pp){ ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); +#ifdef SQLITE_MAX_MEMORY + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ + return 0; + } +#endif + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); @@ -252,7 +258,7 @@ static void mallocWithAlarm(int n, void **pp){ sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } - *pp = p; + return p; } /* @@ -270,7 +276,7 @@ void *sqlite3Malloc(u64 n){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - mallocWithAlarm((int)n, &p); + p = mallocWithAlarm((int)n); sqlite3_mutex_leave(mem0.mutex); }else{ p = sqlite3GlobalConfig.m.xMalloc((int)n); From a6bf20b5872cf8c8490c7efd1f908600a50ec34d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Mar 2017 17:03:11 +0000 Subject: [PATCH 232/292] Fix an error in the SQLITE_MAX_MEMORY implementation resulting from a bad merge. Update the OSSFuzz interface so that it times out after running the byte-code engine for 10 seconds. FossilOrigin-Name: f8560c60d10c0365b33342ab05b5a953987b0471 --- Makefile.in | 1 + Makefile.msc | 2 +- main.mk | 1 + manifest | 21 ++++++++++----------- manifest.uuid | 2 +- src/malloc.c | 3 ++- test/ossfuzz.c | 41 ++++++++++++++++++++++++++++++----------- 7 files changed, 46 insertions(+), 25 deletions(-) diff --git a/Makefile.in b/Makefile.in index d5fa831e6b..ba59b3723f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -566,6 +566,7 @@ SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c DBFUZZ_OPT = diff --git a/Makefile.msc b/Makefile.msc index 317f68fa9b..01307a3e92 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1510,7 +1510,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_ # MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION diff --git a/main.mk b/main.mk index 57f09ff76c..8815f0f701 100644 --- a/main.mk +++ b/main.mk @@ -477,6 +477,7 @@ SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ ST_OPT = -DSQLITE_THREADSAFE=0 diff --git a/manifest b/manifest index 79971ae0b5..4b583d7f1f 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\s-DSQLITE_MAX_MEMORY=N\scompile-time\soption.\s\sThe\sdefault\sis\sno\slimit. -D 2017-03-10T16:22:40.639 -F Makefile.in 5f415e7867296d678fed2e6779aea10c1318b4bc +C Fix\san\serror\sin\sthe\sSQLITE_MAX_MEMORY\simplementation\sresulting\sfrom\sa\sbad\nmerge.\s\sUpdate\sthe\sOSSFuzz\sinterface\sso\sthat\sit\stimes\sout\safter\srunning\nthe\sbyte-code\sengine\sfor\s10\sseconds. +D 2017-03-10T17:03:11.362 +F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 +F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -323,7 +323,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 98f9e673437e28b17f86d07d0749021bb140c152 +F main.mk 0ec10b604f4668f7e85a358954babe75c94dc0d5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -365,7 +365,7 @@ F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 -F src/malloc.c e2b75576ba5587555fa7146cedca437c3d947b9e +F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1007,7 +1007,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c e469138f4be3e92df6173b79b3b216ab6e17b407 +F test/ossfuzz.c f04b9f236e51d4db701bdebe8ac01318c83102a8 F test/ossshell.c d9f1a6f43e7bab45d6be857a5800f5d4a1861db3 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d @@ -1563,8 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 118f5c0564fef70cbd06fc0d9dbb2baec162cc39 77dfe2abdae88dea81217f352d87e5ba2c822715 -R 3b74d97e8af57ebd024f8ef03af4d96f -T +closed 77dfe2abdae88dea81217f352d87e5ba2c822715 +P eabd4ef498a0f0d97d65e321c4d06ab90523ed61 +R 69fed97a3de499803e89b1ea5d13a87c U drh -Z 08daf633daf073490979515612458ee1 +Z 0ae96bef0d12b3fb7ff36d9387e9c8a8 diff --git a/manifest.uuid b/manifest.uuid index dc82ffbe7a..221755c547 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eabd4ef498a0f0d97d65e321c4d06ab90523ed61 \ No newline at end of file +f8560c60d10c0365b33342ab05b5a953987b0471 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 2b903cc61e..6d49107790 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -232,7 +232,8 @@ static void mallocWithAlarm(int n, void **pp){ #ifdef SQLITE_MAX_MEMORY if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ - return 0; + *pp = 0; + return; } #endif diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 3a3e852c34..91b3d1141e 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -6,12 +6,33 @@ #include #include "sqlite3.h" +/* Return the current real-world time in milliseconds since the +** Julian epoch (-4714-11-24). +*/ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* -** Progress handler callback +** Progress handler callback. +** +** The argument is the cutoff-time after which all processing should +** stop. So return non-zero if the cut-off time is exceeded. */ static int progress_handler(void *pReturn) { - return *(int*)pReturn; + sqlite3_int64 iCutoffTime = *(sqlite3_int64*)pReturn; + return timeOfDay()>=iCutoffTime; } #endif @@ -31,13 +52,13 @@ static int exec_handler(void *pCnt, int argc, char **argv, char **namev){ ** fuzzed input. */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - int progressArg = 0; /* 1 causes progress handler abort */ int execCnt = 0; /* Abort row callback when count reaches zero */ char *zErrMsg = 0; /* Error message returned by sqlite_exec() */ sqlite3 *db; /* The database connection */ uint8_t uSelector; /* First byte of input data[] */ int rc; /* Return code from various interfaces */ char *zSql; /* Zero-terminated copy of data[] */ + sqlite3_int64 iCutoff; /* Cutoff timer */ if( size<3 ) return 0; /* Early out if unsufficient data */ @@ -56,16 +77,14 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { if( rc ) return 0; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Bit 0 of the selector enables progress callbacks. Bit 1 is the - ** return code from progress callbacks */ - if( uSelector & 1 ){ - sqlite3_progress_handler(db, 4, progress_handler, (void*)&progressArg); - } + /* Invoke the progress handler every 500 thousand instructions (approximately + ** 20 to 40 times per second) to check to see if we are taking too long. + */ + iCutoff = timeOfDay() + 10000; /* Now + 10 seconds */ + sqlite3_progress_handler(db, 500000, progress_handler, (void*)&iCutoff); #endif - uSelector >>= 1; - progressArg = uSelector & 1; uSelector >>= 1; - /* Bit 2 of the selector enables foreign key constraints */ + /* Bit 1 of the selector enables foreign key constraints */ sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); uSelector >>= 1; From 1ba30db8bddda42163962c75af059007e1efb9ed Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Mar 2017 18:36:34 +0000 Subject: [PATCH 233/292] Remove the rbu_round_trip.tcl script. It is now part of project "test-dbs". FossilOrigin-Name: b5bf2957677e8f2acd7426b302229a966de08fd9 --- ext/rbu/rbu_round_trip.tcl | 205 ------------------------------------- manifest | 13 ++- manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 213 deletions(-) delete mode 100644 ext/rbu/rbu_round_trip.tcl diff --git a/ext/rbu/rbu_round_trip.tcl b/ext/rbu/rbu_round_trip.tcl deleted file mode 100644 index a4f5d579a1..0000000000 --- a/ext/rbu/rbu_round_trip.tcl +++ /dev/null @@ -1,205 +0,0 @@ -########################################################################## -# 2016 Mar 8 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -proc process_cmdline {} { - cmdline::process ::A $::argv { - {make "try to build missing tools"} - {verbose "make more noise"} - database - database2 - } { - This script uses/tests the following tools: - - dbselftest - rbu - sqldiff - sqlite3 - - The user passes the names of two database files - a.db and b.db below - as - arguments. This program: - - 1. Runs [dbselftest --init] against both databases. - 2. Runs [sqldiff --rbu --vtab a.db b.db | sqlite3 .db] to create an - RBU database. - 3. Runs [rbu b.db .db] to patch b.db to a.db. - 4. Runs [sqldiff --table selftest a.db b.db] to check that the selftest - tables are now identical. - 5. Runs [dbselftest] against both databases. - } -} - -########################################################################### -########################################################################### -# Command line options processor. This is generic code that can be copied -# between scripts. -# -namespace eval cmdline { - proc cmdline_error {O E {msg ""}} { - if {$msg != ""} { - puts stderr "Error: $msg" - puts stderr "" - } - - set L [list] - foreach o $O { - if {[llength $o]==1} { - lappend L [string toupper $o] - } - } - - puts stderr "Usage: $::argv0 ?SWITCHES? $L" - puts stderr "" - puts stderr "Switches are:" - foreach o $O { - if {[llength $o]==3} { - foreach {a b c} $o {} - puts stderr [format " -%-15s %s (default \"%s\")" "$a VAL" $c $b] - } elseif {[llength $o]==2} { - foreach {a b} $o {} - puts stderr [format " -%-15s %s" $a $b] - } - } - puts stderr "" - puts stderr $E - exit -1 - } - - proc process {avar lArgs O E} { - upvar $avar A - set zTrailing "" ;# True if ... is present in $O - set lPosargs [list] - - # Populate A() with default values. Also, for each switch in the command - # line spec, set an entry in the idx() array as follows: - # - # {tblname t1 "table name to use"} - # -> [set idx(-tblname) {tblname t1 "table name to use"} - # - # For each position parameter, append its name to $lPosargs. If the ... - # specifier is present, set $zTrailing to the name of the prefix. - # - foreach o $O { - set nm [lindex $o 0] - set nArg [llength $o] - switch -- $nArg { - 1 { - if {[string range $nm end-2 end]=="..."} { - set zTrailing [string range $nm 0 end-3] - } else { - lappend lPosargs $nm - } - } - 2 { - set A($nm) 0 - set idx(-$nm) $o - } - 3 { - set A($nm) [lindex $o 1] - set idx(-$nm) $o - } - default { - error "Error in command line specification" - } - } - } - - # Set explicitly specified option values - # - set nArg [llength $lArgs] - for {set i 0} {$i < $nArg} {incr i} { - set opt [lindex $lArgs $i] - if {[string range $opt 0 0]!="-" || $opt=="--"} break - set c [array names idx "${opt}*"] - if {[llength $c]==0} { cmdline_error $O $E "Unrecognized option: $opt"} - if {[llength $c]>1} { cmdline_error $O $E "Ambiguous option: $opt"} - - if {[llength $idx($c)]==3} { - if {$i==[llength $lArgs]-1} { - cmdline_error $O $E "Option requires argument: $c" - } - incr i - set A([lindex $idx($c) 0]) [lindex $lArgs $i] - } else { - set A([lindex $idx($c) 0]) 1 - } - } - - # Deal with position arguments. - # - set nPosarg [llength $lPosargs] - set nRem [expr $nArg - $i] - if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} { - cmdline_error $O $E - } - for {set j 0} {$j < $nPosarg} {incr j} { - set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]] - } - if {$zTrailing!=""} { - set A($zTrailing) [lrange $lArgs [expr $j+$i] end] - } - } -} ;# namespace eval cmdline -# End of command line options processor. -########################################################################### -########################################################################### - -process_cmdline - -# Check that the specified tool is present. -# -proc check_for_tool {tool} { - if {[file exists $tool]==0 || [file executable $tool]==0} { - puts stderr "Missing $tool... exiting. (run \[make $tool\])" - exit -1 - } -} - -if {$A(make)} { - if {$A(verbose)} { puts "building tools..." } - exec make dbselftest rbu sqlite3 sqldiff -} - -check_for_tool dbselftest -check_for_tool rbu -check_for_tool sqlite3 -check_for_tool sqldiff - -exec ./sqlite3 $A(database) "DROP TABLE selftest;" -exec ./sqlite3 $A(database2) "DROP TABLE selftest;" - -# Run [dbselftest --init] on both databases -if {$A(verbose)} { puts "Running \[dbselftest --init\]" } -exec ./dbselftest --init $A(database) -exec ./dbselftest --init $A(database2) - -# Create an RBU patch. -set tmpname "./rrt-[format %x [expr int(rand()*0x7FFFFFFF)]].db" -if {$A(verbose)} { puts "rbu database is $tmpname" } -exec ./sqldiff --rbu --vtab $A(database2) $A(database) | ./sqlite3 $tmpname - -# Run the [rbu] patch. -if {$A(verbose)} { puts "Running \[rbu]" } -exec ./rbu $A(database2) $tmpname - -set selftest_diff [exec ./sqldiff --table selftest $A(database) $A(database2)] -if {$selftest_diff != ""} { - puts stderr "patching table \"selftest\" failed: $selftest_diff" - exit -1 -} - -# Run [dbselftest] on both databases -if {$A(verbose)} { puts "Running \[dbselftest]" } -exec ./dbselftest $A(database) -exec ./dbselftest $A(database2) - -# Remove the RBU database -file delete $tmpname -puts "round trip test successful." - diff --git a/manifest b/manifest index 4b583d7f1f..d235a22e79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\sSQLITE_MAX_MEMORY\simplementation\sresulting\sfrom\sa\sbad\nmerge.\s\sUpdate\sthe\sOSSFuzz\sinterface\sso\sthat\sit\stimes\sout\safter\srunning\nthe\sbyte-code\sengine\sfor\s10\sseconds. -D 2017-03-10T17:03:11.362 +C Remove\sthe\srbu_round_trip.tcl\sscript.\sIt\sis\snow\spart\sof\sproject\s"test-dbs". +D 2017-03-10T18:36:34.515 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -250,7 +250,6 @@ F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead -F ext/rbu/rbu_round_trip.tcl f3216836bbaeb10d76a2c70b89c13f59eabbd117 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 @@ -1563,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eabd4ef498a0f0d97d65e321c4d06ab90523ed61 -R 69fed97a3de499803e89b1ea5d13a87c -U drh -Z 0ae96bef0d12b3fb7ff36d9387e9c8a8 +P f8560c60d10c0365b33342ab05b5a953987b0471 +R 01deb9b1ef259df8dc49773e784400ec +U dan +Z 416d02cdb56bde7d758b337f12e18e66 diff --git a/manifest.uuid b/manifest.uuid index 221755c547..45f57f4026 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8560c60d10c0365b33342ab05b5a953987b0471 \ No newline at end of file +b5bf2957677e8f2acd7426b302229a966de08fd9 \ No newline at end of file From 891d6b4e9e50e45e732fb759f03088f34b85d604 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Mar 2017 00:46:57 +0000 Subject: [PATCH 234/292] Increase the number of significant digits in floating point literals on ".dump" output from the shell. FossilOrigin-Name: 7359fcacaadc349f520536311dcd1d0b5cea7673 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 8 ++++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d235a22e79..20f373b96b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srbu_round_trip.tcl\sscript.\sIt\sis\snow\spart\sof\sproject\s"test-dbs". -D 2017-03-10T18:36:34.515 +C Increase\sthe\snumber\sof\ssignificant\sdigits\sin\sfloating\spoint\sliterals\son\n".dump"\soutput\sfrom\sthe\sshell. +D 2017-03-11T00:46:57.350 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -399,7 +399,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 4d9be7f0622365029a1f2e0f9628b8e363700ff5 +F src/shell.c df29706f8b19e3b6f695b4f64d6c6963348ca8a4 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8560c60d10c0365b33342ab05b5a953987b0471 -R 01deb9b1ef259df8dc49773e784400ec -U dan -Z 416d02cdb56bde7d758b337f12e18e66 +P b5bf2957677e8f2acd7426b302229a966de08fd9 +R a3ca39ff602fd356af0fc62171a8021d +U drh +Z 21429fdf284b465374d1bf63bafb22c0 diff --git a/manifest.uuid b/manifest.uuid index 45f57f4026..61cc19af5e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5bf2957677e8f2acd7426b302229a966de08fd9 \ No newline at end of file +7359fcacaadc349f520536311dcd1d0b5cea7673 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 378729d09b..04f37809cc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2007,9 +2007,13 @@ static int shell_callback( }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( zSep[0] ) utf8_printf(p->out,"%s",zSep); output_quoted_string(p->out, azArg[i]); - }else if( aiType && (aiType[i]==SQLITE_INTEGER - || aiType[i]==SQLITE_FLOAT) ){ + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ utf8_printf(p->out,"%s%s",zSep, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s%s", zSep, z); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); From 708b22b1e37e0093355911861c192a21f17a7fa6 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Mar 2017 01:56:41 +0000 Subject: [PATCH 235/292] The output of the ".dump" command in the CLI quotes newline and carriage-return characters using "char(10)" and "char(13)". FossilOrigin-Name: 8b2954dd8376e2de985cf5dedeb6eec32c430505 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/shell.c | 52 +++++++++++++++++++++++++++++++++++---------------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 20f373b96b..a01f5a9680 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\snumber\sof\ssignificant\sdigits\sin\sfloating\spoint\sliterals\son\n".dump"\soutput\sfrom\sthe\sshell. -D 2017-03-11T00:46:57.350 +C The\soutput\sof\sthe\s".dump"\scommand\sin\sthe\sCLI\squotes\snewline\sand\scarriage-return\ncharacters\susing\s"char(10)"\sand\s"char(13)". +D 2017-03-11T01:56:41.692 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -399,7 +399,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c df29706f8b19e3b6f695b4f64d6c6963348ca8a4 +F src/shell.c 353f3cebb938099577494326e2853010512e0625 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1562,7 +1562,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b5bf2957677e8f2acd7426b302229a966de08fd9 -R a3ca39ff602fd356af0fc62171a8021d +P 7359fcacaadc349f520536311dcd1d0b5cea7673 +R 21dd742a2e245e17ea4fce05d92294e6 +T *branch * string-quoting-dump +T *sym-string-quoting-dump * +T -sym-trunk * U drh -Z 21429fdf284b465374d1bf63bafb22c0 +Z 185539d207724f873d847ed122ae7c58 diff --git a/manifest.uuid b/manifest.uuid index 61cc19af5e..db4edcf339 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7359fcacaadc349f520536311dcd1d0b5cea7673 \ No newline at end of file +8b2954dd8376e2de985cf5dedeb6eec32c430505 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 04f37809cc..602415f8c9 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1490,32 +1490,52 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ /* ** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. */ static void output_quoted_string(FILE *out, const char *z){ int i; - int nSingle = 0; + char c; setBinaryMode(out, 1); - for(i=0; z[i]; i++){ - if( z[i]=='\'' ) nSingle++; - } - if( nSingle==0 ){ + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ utf8_printf(out,"'%s'",z); }else{ - raw_printf(out,"'"); + int inQuote = 0; + int bStarted = 0; while( *z ){ - for(i=0; z[i] && z[i]!='\''; i++){} - if( i==0 ){ - raw_printf(out,"''"); - z++; - }else if( z[i]=='\'' ){ - utf8_printf(out,"%.*s''",i,z); - z += i+1; - }else{ - utf8_printf(out,"%s",z); + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "'"); + inQuote = 1; + } + utf8_printf(out, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + raw_printf(out, "'"); + continue; + } + if( inQuote ){ + raw_printf(out, "'"); + inQuote = 0; + } + if( c==0 ){ break; } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "char(%d)", c); + bStarted = 1; + } + z += i; } - raw_printf(out,"'"); + if( inQuote ) raw_printf(out, "'"); } setTextMode(out, 1); } From 202230ef5c7ba0eb584aa69be9bed9b8771e553e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Mar 2017 13:02:59 +0000 Subject: [PATCH 236/292] Make sure the translateColumnToCopy() routine in the query planner does not try to access an array that failed to be fully allocated due to a prior OOM. This fixes an issue discovered by OSSFuzz. FossilOrigin-Name: 3299a26160c239255608d1e2b15a221e28b18a3d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 20f373b96b..f5f93fedbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\snumber\sof\ssignificant\sdigits\sin\sfloating\spoint\sliterals\son\n".dump"\soutput\sfrom\sthe\sshell. -D 2017-03-11T00:46:57.350 +C Make\ssure\sthe\stranslateColumnToCopy()\sroutine\sin\sthe\squery\splanner\sdoes\snot\ntry\sto\saccess\san\sarray\sthat\sfailed\sto\sbe\sfully\sallocated\sdue\sto\sa\sprior\sOOM.\nThis\sfixes\san\sissue\sdiscovered\sby\sOSSFuzz. +D 2017-03-11T13:02:59.691 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -479,7 +479,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 -F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 +F src/where.c e815093e5ee039b6b4eb19b646d22deb1a3a523f F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b5bf2957677e8f2acd7426b302229a966de08fd9 -R a3ca39ff602fd356af0fc62171a8021d +P 7359fcacaadc349f520536311dcd1d0b5cea7673 +R b1f6c3d400d5c1eb17e51e0171dc7cb1 U drh -Z 21429fdf284b465374d1bf63bafb22c0 +Z 84001c734b11825e7e05194bb46aad97 diff --git a/manifest.uuid b/manifest.uuid index 61cc19af5e..8d84973a78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7359fcacaadc349f520536311dcd1d0b5cea7673 \ No newline at end of file +3299a26160c239255608d1e2b15a221e28b18a3d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8d0dbb0979..4f65695a3f 100644 --- a/src/where.c +++ b/src/where.c @@ -517,14 +517,16 @@ static LogEst estLog(LogEst N){ ** value stored in its output register. */ static void translateColumnToCopy( - Vdbe *v, /* The VDBE containing code to translate */ + Parse *pParse, /* Parsing context */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ int iRegister, /* The first column is in this register */ int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ ){ + Vdbe *v = pParse->pVdbe; VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); + if( pParse->db->mallocFailed ) return; for(; iStartp1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ @@ -802,7 +804,9 @@ static void constructAutomaticIndex( if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pTabItem->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + testcase( pParse->db->mallocFailed ); + translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, + pTabItem->regResult, 1); sqlite3VdbeGoto(v, addrTop); pTabItem->fg.viaCoroutine = 0; }else{ @@ -4920,8 +4924,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ - translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, + if( pTabItem->fg.viaCoroutine ){ + testcase( pParse->db->mallocFailed ); + translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult, 0); continue; } From 1ed1e616f7930c919d7f7166a232389c2f28edc6 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 12 Mar 2017 19:39:00 +0000 Subject: [PATCH 237/292] Remove an obsolete assert() in the IN operator code generation. FossilOrigin-Name: 18bf6aca2ac86478fd12d5020f3a41cfd2bd2dc3defe2298411f79ad308a6f73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f5f93fedbc..29e9c47ab1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\stranslateColumnToCopy()\sroutine\sin\sthe\squery\splanner\sdoes\snot\ntry\sto\saccess\san\sarray\sthat\sfailed\sto\sbe\sfully\sallocated\sdue\sto\sa\sprior\sOOM.\nThis\sfixes\san\sissue\sdiscovered\sby\sOSSFuzz. -D 2017-03-11T13:02:59.691 +C Remove\san\sobsolete\sassert()\sin\sthe\sIN\soperator\scode\sgeneration. +D 2017-03-12T19:39:00.634 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -351,7 +351,7 @@ F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c 8a29e9b72d4b642189999c41782cd6c5bc43512f +F src/expr.c 7eac40b592672a1f3e0565ac1e66fbb87218436c134d8b2460f989b550e2eb73 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7359fcacaadc349f520536311dcd1d0b5cea7673 -R b1f6c3d400d5c1eb17e51e0171dc7cb1 +P 3299a26160c239255608d1e2b15a221e28b18a3d +R 0614fda50e45b1c812e74d7f5befdd87 U drh -Z 84001c734b11825e7e05194bb46aad97 +Z 926534de7c008338fc48290e21b48744 diff --git a/manifest.uuid b/manifest.uuid index 8d84973a78..64bc83d667 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3299a26160c239255608d1e2b15a221e28b18a3d \ No newline at end of file +18bf6aca2ac86478fd12d5020f3a41cfd2bd2dc3defe2298411f79ad308a6f73 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 2bf76eb0d7..e074f2f443 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2532,7 +2532,6 @@ int sqlite3CodeSubselect( int i; sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); dest.zAffSdst = exprINAffinity(pParse, pExpr); - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ From ba28b5ab0a73e80522d279eb651ac92f4c861bfe Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 12 Mar 2017 20:28:44 +0000 Subject: [PATCH 238/292] Fix a possible NULL pointer dereference in following an OOM error in sqlite3ExprIsInteger(). Problem found by OSS-Fuzz. FossilOrigin-Name: 5ec655e8e817c1ed3bfb2e576745a7cef441494ad7baf1bf9f8895e98ac19c5a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 29e9c47ab1..fe7c037a95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\sassert()\sin\sthe\sIN\soperator\scode\sgeneration. -D 2017-03-12T19:39:00.634 +C Fix\sa\spossible\sNULL\spointer\sdereference\sin\sfollowing\san\sOOM\serror\nin\ssqlite3ExprIsInteger().\sProblem\sfound\sby\sOSS-Fuzz. +D 2017-03-12T20:28:44.701 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -351,7 +351,7 @@ F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c 7eac40b592672a1f3e0565ac1e66fbb87218436c134d8b2460f989b550e2eb73 +F src/expr.c f12a581f342a6fd85d14c31e4fb84f16b3dd107f54d7728dddb62cebc79d7ce1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3299a26160c239255608d1e2b15a221e28b18a3d -R 0614fda50e45b1c812e74d7f5befdd87 +P 18bf6aca2ac86478fd12d5020f3a41cfd2bd2dc3defe2298411f79ad308a6f73 +R ca6b9859462ac91c5f6d6ceb39023c31 U drh -Z 926534de7c008338fc48290e21b48744 +Z af1824315a47164e92b4aa40f4d2923c diff --git a/manifest.uuid b/manifest.uuid index 64bc83d667..d832e1d80a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18bf6aca2ac86478fd12d5020f3a41cfd2bd2dc3defe2298411f79ad308a6f73 \ No newline at end of file +5ec655e8e817c1ed3bfb2e576745a7cef441494ad7baf1bf9f8895e98ac19c5a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index e074f2f443..ce948be69e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1860,6 +1860,7 @@ int sqlite3ExprContainsSubquery(Expr *p){ */ int sqlite3ExprIsInteger(Expr *p, int *pValue){ int rc = 0; + if( p==0 ) return 0; /* Can only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ From bbc017746009e30bbac905bbd67dd1c7e1ac384c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 13:45:29 +0000 Subject: [PATCH 239/292] In the OSSFuzz test module, invoke the progress handler much more frequently so that timeouts are detected punctually even if the test script is running opcodes that individually take a long time (for example, an OP_Function opcode that invokes "randomblob(1.5e6)"). FossilOrigin-Name: f3b6959c04c4ef7b8ff03582b867012a869d52b4a90a0d7ab079ee4c21be5464 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/ossfuzz.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index fe7c037a95..ba1564ad2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spossible\sNULL\spointer\sdereference\sin\sfollowing\san\sOOM\serror\nin\ssqlite3ExprIsInteger().\sProblem\sfound\sby\sOSS-Fuzz. -D 2017-03-12T20:28:44.701 +C In\sthe\sOSSFuzz\stest\smodule,\sinvoke\sthe\sprogress\shandler\smuch\smore\sfrequently\nso\sthat\stimeouts\sare\sdetected\spunctually\seven\sif\sthe\stest\sscript\sis\srunning\nopcodes\sthat\sindividually\stake\sa\slong\stime\s(for\sexample,\san\sOP_Function\sopcode\nthat\sinvokes\s"randomblob(1.5e6)"). +D 2017-03-13T13:45:29.519 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -1006,7 +1006,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c f04b9f236e51d4db701bdebe8ac01318c83102a8 +F test/ossfuzz.c 6dc75478809cfbd4609409a87179ddc2ffaa092e8adb27c1982c5a944a7dd81f F test/ossshell.c d9f1a6f43e7bab45d6be857a5800f5d4a1861db3 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d @@ -1562,7 +1562,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18bf6aca2ac86478fd12d5020f3a41cfd2bd2dc3defe2298411f79ad308a6f73 -R ca6b9859462ac91c5f6d6ceb39023c31 +P 5ec655e8e817c1ed3bfb2e576745a7cef441494ad7baf1bf9f8895e98ac19c5a +R 5ec7c17c414d77e4ccd9fef1ac3d681c U drh -Z af1824315a47164e92b4aa40f4d2923c +Z 5b49012ad9ab98ef47d2e1e6f7e9b1e7 diff --git a/manifest.uuid b/manifest.uuid index d832e1d80a..4616ab6ffc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ec655e8e817c1ed3bfb2e576745a7cef441494ad7baf1bf9f8895e98ac19c5a \ No newline at end of file +f3b6959c04c4ef7b8ff03582b867012a869d52b4a90a0d7ab079ee4c21be5464 \ No newline at end of file diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 91b3d1141e..97d101e17a 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -77,11 +77,13 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { if( rc ) return 0; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Invoke the progress handler every 500 thousand instructions (approximately - ** 20 to 40 times per second) to check to see if we are taking too long. + /* Invoke the progress handler frequently to check to see if we + ** are taking too long. The progress handler will return true + ** (which will block further processing) if more than 10 seconds have + ** elapsed since the start of the test. */ iCutoff = timeOfDay() + 10000; /* Now + 10 seconds */ - sqlite3_progress_handler(db, 500000, progress_handler, (void*)&iCutoff); + sqlite3_progress_handler(db, 10, progress_handler, (void*)&iCutoff); #endif /* Bit 1 of the selector enables foreign key constraints */ From c3becddb75ad4d5cbf28881dad6bf559455ca0b7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Mar 2017 14:30:40 +0000 Subject: [PATCH 240/292] Fix a segfault that could occur following an OOM error in the flattenSubquery() routine. FossilOrigin-Name: c6dda3f752c184f441624c9993e77d5031063d79a0e177b6e25a9886514a742e --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/select.c | 4 +++- test/mallocM.test | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 test/mallocM.test diff --git a/manifest b/manifest index ba1564ad2c..b7307e626a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOSSFuzz\stest\smodule,\sinvoke\sthe\sprogress\shandler\smuch\smore\sfrequently\nso\sthat\stimeouts\sare\sdetected\spunctually\seven\sif\sthe\stest\sscript\sis\srunning\nopcodes\sthat\sindividually\stake\sa\slong\stime\s(for\sexample,\san\sOP_Function\sopcode\nthat\sinvokes\s"randomblob(1.5e6)"). -D 2017-03-13T13:45:29.519 +C Fix\sa\ssegfault\sthat\scould\soccur\sfollowing\san\sOOM\serror\sin\sthe\nflattenSubquery()\sroutine. +D 2017-03-13T14:30:40.789 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -398,7 +398,7 @@ F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f +F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa F src/shell.c df29706f8b19e3b6f695b4f64d6c6963348ca8a4 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -953,6 +953,7 @@ F test/mallocI.test 6c23a71df077fa5d387be90e7e669c5b368ca38a F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 27cb5566a6e5f2d76f9d4aa2eca45524401fd61e F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc +F test/mallocM.test 491001d1e273233048d265ec6d38fdd23745b0284f0c93bc98c94b64451c9c28 F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -1562,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ec655e8e817c1ed3bfb2e576745a7cef441494ad7baf1bf9f8895e98ac19c5a -R 5ec7c17c414d77e4ccd9fef1ac3d681c -U drh -Z 5b49012ad9ab98ef47d2e1e6f7e9b1e7 +P f3b6959c04c4ef7b8ff03582b867012a869d52b4a90a0d7ab079ee4c21be5464 +R 5476ff08c9b02210dda4864e54df5ab0 +U dan +Z 237b4d523d894c49cc7807017d798a94 diff --git a/manifest.uuid b/manifest.uuid index 4616ab6ffc..5bdba86213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3b6959c04c4ef7b8ff03582b867012a869d52b4a90a0d7ab079ee4c21be5464 \ No newline at end of file +c6dda3f752c184f441624c9993e77d5031063d79a0e177b6e25a9886514a742e \ No newline at end of file diff --git a/src/select.c b/src/select.c index d817ebd074..bb055c7894 100644 --- a/src/select.c +++ b/src/select.c @@ -3749,7 +3749,9 @@ static int flattenSubquery( }else{ pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); } - substSelect(pParse, pParent, iParent, pSub->pEList, 0); + if( db->mallocFailed==0 ){ + substSelect(pParse, pParent, iParent, pSub->pEList, 0); + } /* The flattened query is distinct if either the inner or the ** outer query is distinct. diff --git a/test/mallocM.test b/test/mallocM.test new file mode 100644 index 0000000000..85a38acf32 --- /dev/null +++ b/test/mallocM.test @@ -0,0 +1,32 @@ +# 2017 March 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Further OOM tests. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix mallocM + +sqlite3_db_config_lookaside db 0 0 0 + +do_execsql_test 1.0 { + CREATE TABLE t1(x); +} +do_faultsim_test 1 -faults oom-t* -body { + execsql { + SELECT 'abc' FROM ( SELECT 'xyz' FROM t1 WHERE (SELECT 1) ) + } +} -test { + faultsim_test_result {0 {}} +} + +finish_test From 510b7ff53612658ff1a3c6a848779014b3668aeb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 17:37:13 +0000 Subject: [PATCH 241/292] Fix the sqlite3TreeViewSelect() routine so that it works with a null pointer to the Select object. FossilOrigin-Name: 9034cf7efc603864f51e931c7dc4fbbc2d01904e951e78c88d4d80f9936250e8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/treeview.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b7307e626a..cf51919ad1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sthat\scould\soccur\sfollowing\san\sOOM\serror\sin\sthe\nflattenSubquery()\sroutine. -D 2017-03-13T14:30:40.789 +C Fix\sthe\ssqlite3TreeViewSelect()\sroutine\sso\sthat\sit\sworks\swith\sa\snull\spointer\nto\sthe\sSelect\sobject. +D 2017-03-13T17:37:13.752 F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -459,7 +459,7 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c dc748c5afcb9e5beb3ef5651bc99a4622e30f6a1 -F src/treeview.c 4e44ade3bfe59d82005039f72e09333ce2b4162c +F src/treeview.c 84d0ac737e1231702679f0289180021e19c5cc186ec413e8dcb704a887c76ec8 F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3b6959c04c4ef7b8ff03582b867012a869d52b4a90a0d7ab079ee4c21be5464 -R 5476ff08c9b02210dda4864e54df5ab0 -U dan -Z 237b4d523d894c49cc7807017d798a94 +P c6dda3f752c184f441624c9993e77d5031063d79a0e177b6e25a9886514a742e +R 21d5fb07d70376e4a3754546b17385a3 +U drh +Z aeef669a336ea6338a3e27ef9df24400 diff --git a/manifest.uuid b/manifest.uuid index 5bdba86213..f701b94f3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6dda3f752c184f441624c9993e77d5031063d79a0e177b6e25a9886514a742e \ No newline at end of file +9034cf7efc603864f51e931c7dc4fbbc2d01904e951e78c88d4d80f9936250e8 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 0ea512b5c6..61d4626a22 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -126,6 +126,10 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ int n = 0; int cnt = 0; + if( p==0 ){ + sqlite3TreeViewLine(pView, "nil-SELECT"); + return; + } pView = sqlite3TreeViewPush(pView, moreToFollow); if( p->pWith ){ sqlite3TreeViewWith(pView, p->pWith, 1); From 688633cb9eb400d8c69cbbcd99a4410a810bf0fb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 19:26:34 +0000 Subject: [PATCH 242/292] Infrastructure for an extension C-library to implement sqlite3_db_dump() and a corresponding "dbdump" command-line utility - both of which do the same work as the ".dump" command of the CLI. FossilOrigin-Name: 74c5ace498f72d7f5495203678bedd0bc540211131a4e4db7b62115d5322a288 --- Makefile.in | 4 ++ ext/misc/dbdump.c | 103 ++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 4 ++ manifest | 19 +++++---- manifest.uuid | 2 +- 5 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 ext/misc/dbdump.c diff --git a/Makefile.in b/Makefile.in index ba59b3723f..0b40cb458b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1154,6 +1154,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo + $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ + $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) + showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c new file mode 100644 index 0000000000..b7fe595e29 --- /dev/null +++ b/ext/misc/dbdump.c @@ -0,0 +1,103 @@ +/* +** 2016-03-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a C-language subroutine that converts the content +** of an SQLite database into UTF-8 text SQL statements that can be used +** to exactly recreate the original database. +** +** A prototype of the implemented subroutine is this: +** +** int sqlite3_db_dump( +** sqlite3 *db, +** const char *zSchema, +** const char *zTable, +** void (*xCallback)(void*, const char*), +** void *pArg +** ); +** +** The db parameter is the database connection. zSchema is the schema within +** that database which is to be dumped. Usually the zSchema is "main" but +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then +** only the content of that one table is dumped. If zTable is NULL, then all +** tables are dumped. +** +** The generate text is passed to xCallback() in multiple calls. The second +** argument to xCallback() is a copy of the pArg parameter. The first +** argument is some of the output text that this routine generates. The +** signature to xCallback() is designed to make it compatible with fputs(). +** +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error +** code if it encounters a problem. +** +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine +** is included so that this routine becomes a command-line utility. The +** command-line utility takes two or three arguments which are the name +** of the database file, the schema, and optionally the table, forming the +** first three arguments of a single call to the library routine. +*/ +#include "sqlite3.h" + +/* +** Convert an SQLite database into SQL statements that will recreate that +** database. +*/ +int sqlite3_db_dump( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which schema to dump. Usually "main". */ + const char *zTable, /* Which table to dump. NULL means everything. */ + int (*xCallback)(const char*,void*), /* Output sent to this callback */ + void *pArg /* Second argument of the callback */ +){ + return SQLITE_OK; +} + + + +/* The generic subroutine is above. The code the follows implements +** the command-line interface. +*/ +#ifdef DBDUMP_STANDALONE +#include + +/* +** Command-line interface +*/ +int main(int argc, char **argv){ + sqlite3 *db; + const char *zDb; + const char *zSchema; + const char *zTable = 0; + int rc; + + if( argc<2 || argc>4 ){ + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); + return 1; + } + zDb = argv[1]; + zSchema = argc>=3 ? argv[2] : "main"; + zTable = argc==4 ? argv[3] : 0; + + rc = sqlite3_open(zDb, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + rc = sqlite3_db_dump(db, zSchema, zTable, + (int(*)(const char*,void*))fputs, (void*)stdout); + if( rc ){ + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); + } + sqlite3_close(db); + return rc!=SQLITE_OK; +} +#endif /* DBDUMP_STANDALONE */ diff --git a/main.mk b/main.mk index 8815f0f701..54f223bb56 100644 --- a/main.mk +++ b/main.mk @@ -761,6 +761,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o + $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ + $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) + # Rules to build the 'testfixture' application. # TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 diff --git a/manifest b/manifest index b74ca835a2..4fc996e4ef 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C In\sthe\soutput\sof\sthe\s".dump"\scommand\sin\sthe\sCLI,\squote\snewline\sand\ncarriage-return\scharacters\susing\sthe\schar()\sfunction,\sso\sthat\sthey\sdo\snot\nget\seaten\sby\send-of-line\sprocessing\slogic\sin\sthe\sOS\sor\sin\sother\scommand-line\nutilities\sand/or\slibraries. -D 2017-03-13T18:24:06.748 -F Makefile.in 2dae2a56457c2885425a480e1053de8096aff924 +C Infrastructure\sfor\san\sextension\sC-library\sto\simplement\ssqlite3_db_dump()\sand\na\scorresponding\s"dbdump"\scommand-line\sutility\s-\sboth\sof\swhich\sdo\sthe\ssame\nwork\sas\sthe\s".dump"\scommand\sof\sthe\sCLI. +D 2017-03-13T19:26:34.140 +F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -211,6 +211,7 @@ F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d +F ext/misc/dbdump.c 5777b95fe1797a7fce841f4cb88bc30f66b8a3c929e988b1d3b3cf4aab044739 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 @@ -322,7 +323,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 0ec10b604f4668f7e85a358954babe75c94dc0d5 +F main.mk 9abb506e717887d57f754bae139b85c1a06d6f2ac25b589f3e792e310567f278 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1563,8 +1564,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9034cf7efc603864f51e931c7dc4fbbc2d01904e951e78c88d4d80f9936250e8 8b2954dd8376e2de985cf5dedeb6eec32c430505 -R 73c5fbe608fb0e20ee39173a6198ea04 -T +closed 8b2954dd8376e2de985cf5dedeb6eec32c430505 +P 68f6dc7af1013f296a11db14c007cc13cc3fe56832848bfed835ed8f74dcc676 +R 26910a3d8e71b6c159f5bba8a9a5daac +T *branch * dbdump +T *sym-dbdump * +T -sym-trunk * U drh -Z 5e4a79f0eabba6013ee57f9716ad3797 +Z 0266a8ae90d3841510ebcee8fa626828 diff --git a/manifest.uuid b/manifest.uuid index 35073a3308..f9e622d35f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68f6dc7af1013f296a11db14c007cc13cc3fe56832848bfed835ed8f74dcc676 \ No newline at end of file +74c5ace498f72d7f5495203678bedd0bc540211131a4e4db7b62115d5322a288 \ No newline at end of file From 473af4fe03643d1cc771a3e394eb8579c3fbf61a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 21:26:41 +0000 Subject: [PATCH 243/292] First draft of the complete dbdump.c library. FossilOrigin-Name: 84ea4fcc52d0af02648c52989c2e69f4961071e1620382555ec59a39161a7a7d --- ext/misc/dbdump.c | 614 +++++++++++++++++++++++++++++++++++++++++++++- manifest | 15 +- manifest.uuid | 2 +- 3 files changed, 619 insertions(+), 12 deletions(-) diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c index b7fe595e29..ce51eba0a0 100644 --- a/ext/misc/dbdump.c +++ b/ext/misc/dbdump.c @@ -12,7 +12,7 @@ ** ** This file implements a C-language subroutine that converts the content ** of an SQLite database into UTF-8 text SQL statements that can be used -** to exactly recreate the original database. +** to exactly recreate the original database. ROWID values are preserved. ** ** A prototype of the implemented subroutine is this: ** @@ -45,6 +45,574 @@ ** first three arguments of a single call to the library routine. */ #include "sqlite3.h" +#include +#include +#include + +/* +** The state of the dump process. +*/ +typedef struct DState DState; +struct DState { + sqlite3 *db; /* The database connection */ + int nErr; /* Number of errors seen so far */ + int rc; /* Error code */ + int writableSchema; /* True if in writable_schema mode */ + int (*xCallback)(const char*,void*); /* Send output here */ + void *pArg; /* Argument to xCallback() */ +}; + +/* +** A variable length string to which one can append text. +*/ +typedef struct DText DText; +struct DText { + char *z; /* The text */ + int n; /* Number of bytes of content in z[] */ + int nAlloc; /* Number of bytes allocated to z[] */ +}; + +/* +** Initialize and destroy a DText object +*/ +static void initText(DText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(DText *p){ + sqlite3_free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(DText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + len + 20; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + freeText(p); + return; + } + p->z = zNew; + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} + + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(DState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt = 0; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = 1; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + azCol[0] = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + char **azNew; + nAlloc = nAlloc*2 + nCol + 10; + azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azNew==0 ) goto col_oom; + azCol = azNew; + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( azCol[nCol]==0 ) goto col_oom; + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + if( zSql==0 ) goto col_oom; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + int rc; + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; + +col_oom: + sqlite3_finalize(pStmt); + freeColumnList(azCol); + p->nErr++; + p->rc = SQLITE_NOMEM; + return 0; +} + +/* +** Send mprintf-formatted content to the output callback. +*/ +static void output_formatted(DState *p, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_mprintf(zFormat, ap); + va_end(ap); + p->xCallback(z, p->pArg); + sqlite3_free(z); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. +*/ +static void output_quoted_string(DState *p, const unsigned char *z){ + int i; + char c; + int inQuote = 0; + int bStarted = 0; + + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + output_formatted(p, "'%s'", z); + return; + } + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) p->xCallback("||", p->pArg); + p->xCallback("'", p->pArg); + inQuote = 1; + } + output_formatted(p, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + p->xCallback("'", p->pArg); + continue; + } + if( inQuote ){ + p->xCallback("'", p->pArg); + inQuote = 0; + } + if( c==0 ){ + break; + } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) p->xCallback("||", p->pArg); + output_formatted(p, "char(%d)", c); + bStarted = 1; + } + z += i; + } + if( inQuote ) p->xCallback("'", p->pArg); +} + +/* +** This is an sqlite3_exec callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + DState *p = (DState*)pArg; + sqlite3_stmt *pStmt; + + (void)azCol; + if( nArg!=3 ) return 1; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + + if( strcmp(zTable, "sqlite_sequence")==0 ){ + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ + p->xCallback("ANALYZE sqlite_master;\n", p->pArg); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + if( !p->writableSchema ){ + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); + p->writableSchema = 1; + } + output_formatted(p, + "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + return 0; + }else{ + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); + p->xCallback(zSql+13, p->pArg); + }else{ + p->xCallback(zSql, p->pArg); + } + p->xCallback(";\n", p->pArg); + } + + if( strcmp(zType, "table")==0 ){ + DText sSelect; + DText sTable; + char **azCol; + int i; + int nCol; + + azCol = tableColumnList(p, zTable); + if( azCol==0 ) return 0; + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); + } + appendText(&sTable, ")", 0); + } + appendText(&sTable, " VALUES(", 0); + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + nCol = i-1; + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + p->nErr++; + if( p->rc==SQLITE_OK ) p->rc = rc; + }else{ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->xCallback(sTable.z, p->pArg); + for(i=0; ixCallback(",", p->pArg); + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_INTEGER: { + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_column_double(pStmt,i); + output_formatted(p, "%!.20g", r); + break; + } + case SQLITE_NULL: { + p->xCallback("NULL", p->pArg); + break; + } + case SQLITE_TEXT: { + output_quoted_string(p, sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + int nByte = sqlite3_column_bytes(pStmt,i); + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); + int j; + p->xCallback("x'", p->pArg); + for(j=0; j>4)&15]; + zWord[1] = "0123456789abcdef"[a[j]&15]; + zWord[2] = 0; + p->xCallback(zWord, p->pArg); + } + p->xCallback("'", p->pArg); + break; + } + } + } + p->xCallback(");\n", p->pArg); + } + } + sqlite3_finalize(pStmt); + freeText(&sTable); + freeText(&sSelect); + } + return 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static void output_sql_from_query( + DState *p, /* Query context */ + const char *zSelect, /* SELECT statement to extract content */ + ... +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + char *zSql; + va_list ap; + va_start(ap, zSelect); + zSql = sqlite3_mprintf(zSelect, ap); + va_end(ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + p->nErr++; + return; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK || !pSelect ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + p->nErr++; + return; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + p->xCallback(z, p->pArg); + for(i=1; ixCallback(",", p->pArg); + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + p->xCallback("\n;\n", p->pArg); + }else{ + p->xCallback(";\n", p->pArg); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static void run_schema_dump_query( + DState *p, + const char *zQuery, + ... +){ + char *zErr = 0; + char *z; + va_list ap; + va_start(ap, zQuery); + z = sqlite3_mprintf(zQuery, ap); + va_end(ap); + sqlite3_exec(p->db, z, dump_callback, p, &zErr); + sqlite3_free(z); + if( zErr ){ + output_formatted(p, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + p->nErr++; + zErr = 0; + } +} /* ** Convert an SQLite database into SQL statements that will recreate that @@ -57,7 +625,49 @@ int sqlite3_db_dump( int (*xCallback)(const char*,void*), /* Output sent to this callback */ void *pArg /* Second argument of the callback */ ){ - return SQLITE_OK; + DState x; + memset(&x, 0, sizeof(x)); + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( x.rc ) return x.rc; + x.db = db; + x.xCallback = xCallback; + x.pArg = pArg; + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); + if( zTable==0 ){ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", + zSchema + ); + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE name=='sqlite_sequence'", zSchema + ); + output_sql_from_query(&x, + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); + }else{ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" + " AND sql NOT NULL", + zSchema, zTable + ); + output_sql_from_query(&x, + "SELECT sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name=%Q COLLATE nocase", + zSchema, zTable + ); + } + if( x.writableSchema ){ + xCallback("PRAGMA writable_schema=OFF;\n", pArg); + } + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); + sqlite3_exec(db, "COMMIT", 0, 0, 0); + return x.rc; } diff --git a/manifest b/manifest index 4fc996e4ef..dfbee1c83e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Infrastructure\sfor\san\sextension\sC-library\sto\simplement\ssqlite3_db_dump()\sand\na\scorresponding\s"dbdump"\scommand-line\sutility\s-\sboth\sof\swhich\sdo\sthe\ssame\nwork\sas\sthe\s".dump"\scommand\sof\sthe\sCLI. -D 2017-03-13T19:26:34.140 +C First\sdraft\sof\sthe\scomplete\sdbdump.c\slibrary. +D 2017-03-13T21:26:41.643 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -211,7 +211,7 @@ F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d -F ext/misc/dbdump.c 5777b95fe1797a7fce841f4cb88bc30f66b8a3c929e988b1d3b3cf4aab044739 +F ext/misc/dbdump.c 819eb33f6ff788a466525557f7ea3742078853cc794b2eb0799a254dd6da81e2 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 @@ -1564,10 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68f6dc7af1013f296a11db14c007cc13cc3fe56832848bfed835ed8f74dcc676 -R 26910a3d8e71b6c159f5bba8a9a5daac -T *branch * dbdump -T *sym-dbdump * -T -sym-trunk * +P 74c5ace498f72d7f5495203678bedd0bc540211131a4e4db7b62115d5322a288 +R dcbc2252d3d150af6e3faa70158994bd U drh -Z 0266a8ae90d3841510ebcee8fa626828 +Z 9a73382dae2ea2eb6bda6a3d74fd54dd diff --git a/manifest.uuid b/manifest.uuid index f9e622d35f..0597c414ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74c5ace498f72d7f5495203678bedd0bc540211131a4e4db7b62115d5322a288 \ No newline at end of file +84ea4fcc52d0af02648c52989c2e69f4961071e1620382555ec59a39161a7a7d \ No newline at end of file From d3bc75fb941cdadf06bd8d5424ec8d856c89aabe Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 21:49:48 +0000 Subject: [PATCH 244/292] Fixes to the dump logic. All appears to be working in preliminary tests. FossilOrigin-Name: 007b11e301d444361c8eff8734dc2fb968a64343c177ff30cec74a0cf76099e8 --- ext/misc/dbdump.c | 16 ++++++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c index ce51eba0a0..db581a8325 100644 --- a/ext/misc/dbdump.c +++ b/ext/misc/dbdump.c @@ -224,7 +224,6 @@ static char **tableColumnList(DState *p, const char *zTab){ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc ) return 0; - azCol[0] = 0; while( sqlite3_step(pStmt)==SQLITE_ROW ){ if( nCol>=nAlloc-2 ){ char **azNew; @@ -232,6 +231,7 @@ static char **tableColumnList(DState *p, const char *zTab){ azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); if( azNew==0 ) goto col_oom; azCol = azNew; + azCol[0] = 0; } azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); if( azCol[nCol]==0 ) goto col_oom; @@ -317,7 +317,7 @@ static void output_formatted(DState *p, const char *zFormat, ...){ va_list ap; char *z; va_start(ap, zFormat); - z = sqlite3_mprintf(zFormat, ap); + z = sqlite3_vmprintf(zFormat, ap); va_end(ap); p->xCallback(z, p->pArg); sqlite3_free(z); @@ -430,10 +430,13 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ azCol = tableColumnList(p, zTable); if( azCol==0 ) return 0; + initText(&sTable); + appendText(&sTable, "INSERT INTO ", 0); + /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - initText(&sTable); appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" ** instead of the usual "INSERT INTO tab VALUES(...)". @@ -462,7 +465,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ appendText(&sSelect, ",", 0); } } - nCol = i-1; + nCol = i; + if( azCol[0]==0 ) nCol--; freeColumnList(azCol); appendText(&sSelect, " FROM ", 0); appendText(&sSelect, zTable, quoteChar(zTable)); @@ -545,7 +549,7 @@ static void output_sql_from_query( char *zSql; va_list ap; va_start(ap, zSelect); - zSql = sqlite3_mprintf(zSelect, ap); + zSql = sqlite3_vmprintf(zSelect, ap); va_end(ap); if( zSql==0 ){ p->rc = SQLITE_NOMEM; @@ -602,7 +606,7 @@ static void run_schema_dump_query( char *z; va_list ap; va_start(ap, zQuery); - z = sqlite3_mprintf(zQuery, ap); + z = sqlite3_vmprintf(zQuery, ap); va_end(ap); sqlite3_exec(p->db, z, dump_callback, p, &zErr); sqlite3_free(z); diff --git a/manifest b/manifest index dfbee1c83e..57159e7a5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sdraft\sof\sthe\scomplete\sdbdump.c\slibrary. -D 2017-03-13T21:26:41.643 +C Fixes\sto\sthe\sdump\slogic.\s\sAll\sappears\sto\sbe\sworking\sin\spreliminary\stests. +D 2017-03-13T21:49:48.351 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 @@ -211,7 +211,7 @@ F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d -F ext/misc/dbdump.c 819eb33f6ff788a466525557f7ea3742078853cc794b2eb0799a254dd6da81e2 +F ext/misc/dbdump.c 34174d537318027b159e3341105866c05a4149376ede59988c82280bed144e6d F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 74c5ace498f72d7f5495203678bedd0bc540211131a4e4db7b62115d5322a288 -R dcbc2252d3d150af6e3faa70158994bd +P 84ea4fcc52d0af02648c52989c2e69f4961071e1620382555ec59a39161a7a7d +R de364a9f85fa4899d644a897efb72e80 U drh -Z 9a73382dae2ea2eb6bda6a3d74fd54dd +Z 55e9f45d2725daddcac9a1569ce24da7 diff --git a/manifest.uuid b/manifest.uuid index 0597c414ba..db7f312b50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84ea4fcc52d0af02648c52989c2e69f4961071e1620382555ec59a39161a7a7d \ No newline at end of file +007b11e301d444361c8eff8734dc2fb968a64343c177ff30cec74a0cf76099e8 \ No newline at end of file From 4b34eb3bf5ad83be4ed86a4b5bef65945dfec95c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Mar 2017 22:02:01 +0000 Subject: [PATCH 245/292] Add dbdump.exe to the MSVC makefile. FossilOrigin-Name: 59241a50ad1d6fe9f5804b73b3467dcd407d359ccbdcb9d72f2f6d94c90c1f40 --- Makefile.msc | 4 ++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 01307a3e92..ecaa837abb 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2178,6 +2178,10 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +dbdump.exe: $(TOP)\ext\misc\dbdump.c sqlite3.c + $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c sqlite3.c \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) + testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c diff --git a/manifest b/manifest index 57159e7a5c..46a6ae6e2d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fixes\sto\sthe\sdump\slogic.\s\sAll\sappears\sto\sbe\sworking\sin\spreliminary\stests. -D 2017-03-13T21:49:48.351 +C Add\sdbdump.exe\sto\sthe\sMSVC\smakefile. +D 2017-03-13T22:02:01.034 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 9020fa41eb91f657ae0cc44145d0a2f3af520860 +F Makefile.msc 5f1194a315c341c14870534652afd34bfa701219e5994fa0ee0bac91389fb740 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84ea4fcc52d0af02648c52989c2e69f4961071e1620382555ec59a39161a7a7d -R de364a9f85fa4899d644a897efb72e80 +P 007b11e301d444361c8eff8734dc2fb968a64343c177ff30cec74a0cf76099e8 +R 6996c48c318cda176124bead4ce6093a U drh -Z 55e9f45d2725daddcac9a1569ce24da7 +Z a94f031011fffd33ab6cd5678ecde717 diff --git a/manifest.uuid b/manifest.uuid index db7f312b50..bf39e3c0d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -007b11e301d444361c8eff8734dc2fb968a64343c177ff30cec74a0cf76099e8 \ No newline at end of file +59241a50ad1d6fe9f5804b73b3467dcd407d359ccbdcb9d72f2f6d94c90c1f40 \ No newline at end of file From 943d825b544efed6ff58bbfaf4b32d3c399c17d1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 14 Mar 2017 15:27:56 +0000 Subject: [PATCH 246/292] Updates to the MSVC makefiles. FossilOrigin-Name: 1e4b9e7c9ee32b985c440bf3eb65bbdf28055566e66b6feba438143a3ad8dd07 --- Makefile.msc | 8 ++++---- autoconf/Makefile.msc | 1 + manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index ecaa837abb..1dfc6bc49d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2178,9 +2178,9 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) -dbdump.exe: $(TOP)\ext\misc\dbdump.c sqlite3.c - $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c sqlite3.c \ - /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) +dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c @@ -2260,7 +2260,7 @@ clean: del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL - del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 32ef1c5434..8c13778fc8 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -644,6 +644,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF diff --git a/manifest b/manifest index a677fe2bf1..3813dabd09 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\sdbdump.c\sextension\sthat\simplements\sfunctionality\ssimilar\sto\sthe\n".dump"\scommand\sof\sthe\sCLI,\sthough\sin\sa\sseparate\slibrary. -D 2017-03-13T22:10:05.684 +C Updates\sto\sthe\sMSVC\smakefiles. +D 2017-03-14T15:27:56.153 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 5f1194a315c341c14870534652afd34bfa701219e5994fa0ee0bac91389fb740 +F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc 3de603bcec5c07462801a20fd7dc1b871400af99 +F autoconf/Makefile.msc 92e3d7fd64dc5c23a23d33824ee4709aaea00a6331c4bb41a1aadcf6600a49f7 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc @@ -1564,8 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68f6dc7af1013f296a11db14c007cc13cc3fe56832848bfed835ed8f74dcc676 59241a50ad1d6fe9f5804b73b3467dcd407d359ccbdcb9d72f2f6d94c90c1f40 -R 6996c48c318cda176124bead4ce6093a -T +closed 59241a50ad1d6fe9f5804b73b3467dcd407d359ccbdcb9d72f2f6d94c90c1f40 -U drh -Z 2f55f00bb27f982c54090f39d62e41f0 +P 2b9980a292d7f0d995b77d9b3fb217bb2206f6d452e036ac502fa06561af4c12 +R 8e14f9dee9f474ef44485ac6e8f73fc9 +U mistachkin +Z b0310bb4b1f32ecae5a830c4ceeca98b diff --git a/manifest.uuid b/manifest.uuid index 52fbd6fac9..6341cc740e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b9980a292d7f0d995b77d9b3fb217bb2206f6d452e036ac502fa06561af4c12 \ No newline at end of file +1e4b9e7c9ee32b985c440bf3eb65bbdf28055566e66b6feba438143a3ad8dd07 \ No newline at end of file From 9c4527e57d7f6a0cd2b108ea6fd1fdc5f3e851c1 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Mar 2017 13:47:39 +0000 Subject: [PATCH 247/292] Improved header comment on the shathree.c extension. No changes to code. FossilOrigin-Name: 84f2e3d5f611b35de16684956d842df6c93d858e8187f17eb27452758a752c57 --- ext/misc/shathree.c | 4 ++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index 52bfdb0363..612e395ccb 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -21,6 +21,10 @@ ** ** The sha3_query(Y) function evalutes all queries in the SQL statements of Y ** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/manifest b/manifest index 3813dabd09..ef98d84342 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sMSVC\smakefiles. -D 2017-03-14T15:27:56.153 +C Improved\sheader\scomment\son\sthe\sshathree.c\sextension.\s\sNo\schanges\sto\scode. +D 2017-03-15T13:47:39.337 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -226,7 +226,7 @@ F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 -F ext/misc/shathree.c 38aa4c3c3d3c3fc34b8ea367bed299d8d8a224b2 +F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2b9980a292d7f0d995b77d9b3fb217bb2206f6d452e036ac502fa06561af4c12 -R 8e14f9dee9f474ef44485ac6e8f73fc9 -U mistachkin -Z b0310bb4b1f32ecae5a830c4ceeca98b +P 1e4b9e7c9ee32b985c440bf3eb65bbdf28055566e66b6feba438143a3ad8dd07 +R 54add779c916bea2d103fd3d0493702d +U drh +Z 250a0bd95a8a1e19df805d19a289213d diff --git a/manifest.uuid b/manifest.uuid index 6341cc740e..a77693e8fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e4b9e7c9ee32b985c440bf3eb65bbdf28055566e66b6feba438143a3ad8dd07 \ No newline at end of file +84f2e3d5f611b35de16684956d842df6c93d858e8187f17eb27452758a752c57 \ No newline at end of file From 7553c82894a86bcd0cdc6a1e8a3e334f5a761394 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Mar 2017 14:04:03 +0000 Subject: [PATCH 248/292] Make "PRAGMA temp.synchronous=N" a no-op to ensure that the TEMP schema always has synchronous=OFF. This fixes an issue discovered by OSS-Fuzz. FossilOrigin-Name: bcf6bb08d8b07d3c4567bcd367f8d4011ce8baad28c25d4e8870bdf347dd48bd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ef98d84342..9b3d61974d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sheader\scomment\son\sthe\sshathree.c\sextension.\s\sNo\schanges\sto\scode. -D 2017-03-15T13:47:39.337 +C Make\s"PRAGMA\stemp.synchronous=N"\sa\sno-op\sto\sensure\sthat\sthe\sTEMP\sschema\nalways\shas\ssynchronous=OFF.\s\sThis\sfixes\san\sissue\sdiscovered\sby\sOSS-Fuzz. +D 2017-03-15T14:04:03.559 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -392,7 +392,7 @@ F src/parse.y 48b03113704ee8bd78ee6996d81de7fbee22e105 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c b7bf4f1dd9cdcdd3f72f0bdbce3502d69d0f48ba +F src/pragma.c bfaa7e5cbfc8eacaa0f3611d5ec2dca1339d6d301f1e9b429b49ca460d794a60 F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e4b9e7c9ee32b985c440bf3eb65bbdf28055566e66b6feba438143a3ad8dd07 -R 54add779c916bea2d103fd3d0493702d +P 84f2e3d5f611b35de16684956d842df6c93d858e8187f17eb27452758a752c57 +R 6e100ddef7b1acc3bf828911d35ed937 U drh -Z 250a0bd95a8a1e19df805d19a289213d +Z 66f9c74ac924ecc8772889a54467a4f5 diff --git a/manifest.uuid b/manifest.uuid index a77693e8fa..b6378adfd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84f2e3d5f611b35de16684956d842df6c93d858e8187f17eb27452758a752c57 \ No newline at end of file +bcf6bb08d8b07d3c4567bcd367f8d4011ce8baad28c25d4e8870bdf347dd48bd \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 2901f4bdc5..488a1d194c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1015,7 +1015,7 @@ void sqlite3Pragma( if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); - }else{ + }else if( iDb!=1 ){ int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; From a5d75ba9ead98c7721014019f5ff33a20dbbdfbb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Mar 2017 14:20:34 +0000 Subject: [PATCH 249/292] Mention the ".selftest" command in the ".help" output of the CLI. FossilOrigin-Name: 37f766dbad1f99ff86dd1b771bf443036e928e5b4d8abe55bbe4acf3362c7be2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9b3d61974d..3d90b0cede 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\s"PRAGMA\stemp.synchronous=N"\sa\sno-op\sto\sensure\sthat\sthe\sTEMP\sschema\nalways\shas\ssynchronous=OFF.\s\sThis\sfixes\san\sissue\sdiscovered\sby\sOSS-Fuzz. -D 2017-03-15T14:04:03.559 +C Mention\sthe\s".selftest"\scommand\sin\sthe\s".help"\soutput\sof\sthe\sCLI. +D 2017-03-15T14:20:34.151 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -400,7 +400,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa -F src/shell.c 353f3cebb938099577494326e2853010512e0625 +F src/shell.c 77054c021069ec0b65d3d620aab031f97c59b4e42ac7c31544ea68933b581104 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84f2e3d5f611b35de16684956d842df6c93d858e8187f17eb27452758a752c57 -R 6e100ddef7b1acc3bf828911d35ed937 +P bcf6bb08d8b07d3c4567bcd367f8d4011ce8baad28c25d4e8870bdf347dd48bd +R 3a48c86458783a20973121829bd123da U drh -Z 66f9c74ac924ecc8772889a54467a4f5 +Z 148bc384ce73e7778d1614d5fa3b82a8 diff --git a/manifest.uuid b/manifest.uuid index b6378adfd0..3ea504fab7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcf6bb08d8b07d3c4567bcd367f8d4011ce8baad28c25d4e8870bdf347dd48bd \ No newline at end of file +37f766dbad1f99ff86dd1b771bf443036e928e5b4d8abe55bbe4acf3362c7be2 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 602415f8c9..e7652f8e67 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3169,6 +3169,7 @@ static char zHelp[] = ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" ".schema ?PATTERN? Show the CREATE statements matching PATTERN\n" " Add --indent for pretty-printing\n" + ".selftest ?--init? Run tests defined in the SELFTEST table\n" ".separator COL ?ROW? Change the column separator and optionally the row\n" " separator for both the output mode and .import\n" #if defined(SQLITE_ENABLE_SESSION) From 6295ff1d3eecfc746632b54b543771b97c6e2bab Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Mar 2017 19:11:29 +0000 Subject: [PATCH 250/292] Add the "Obtaining The Code" subsection in the top-level README.md file. No changes to code. FossilOrigin-Name: b1b1aa8b69aa80c83aec3380565f0b4ec0b6a6e033537becee098872da362e9a --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++----------- manifest | 12 ++++----- manifest.uuid | 2 +- 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index dbc020574e..7b860c211e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,48 @@ If you are reading this on a Git mirror someplace, you are doing it wrong. The [official repository](https://www.sqlite.org/src/) is better. Go there now. +## Obtaining The Code + +SQLite sources are managed using the +[Fossil](https://www.fossil-scm.org/), a distributed version control system +that was specifically designed to support SQLite development. +If you do not want to use Fossil, you can download tarballs or ZIP +archives as follows: + + * Lastest trunk check-in: + or + . + + * Latest release: + or + . + + * For other check-ins, substitute an appropriate branch name or + tag or hash prefix for "release" in the URLs of the previous + bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) + to locate the check-in desired, click on its information page link, + then click on the "Tarball" or "ZIP Archive" links on the information + page. + +If you do want to use Fossil to check out the source tree, +first install Fossil version 2.0 or later. +(Source tarballs and precompiled binaries available +[here](https://www.fossil-scm.org/fossil/uv/download.html).) +Then run commands like this: + + mkdir ~/sqlite + cd ~/sqlite + fossil clone https://www.sqlite.org/src sqlite.fossil + fossil open sqlite.fossil + +After setting up a repository using the steps above, you can always +update to the lastest version using: + + fossil update trunk ;# latest trunk check-in + fossil update release ;# latest official release + +Or type "fossil ui" to get a web-based user interface. + ## Compiling First create a directory in which to place @@ -18,13 +60,13 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Run the makefile. - make sqlite3.c ;# Build the "amalgamation" source file - make test ;# Run some tests (requires Tcl) + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) See the makefile for additional targets. @@ -43,13 +85,13 @@ with the provided "Makefile.msc" to build one of the supported targets. For example: - mkdir bld - cd bld - nmake /f Makefile.msc TOP=..\sqlite - nmake /f Makefile.msc sqlite3.c TOP=..\sqlite - nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite - nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite - nmake /f Makefile.msc test TOP=..\sqlite + mkdir bld + cd bld + nmake /f Makefile.msc TOP=..\sqlite + nmake /f Makefile.msc sqlite3.c TOP=..\sqlite + nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite + nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite + nmake /f Makefile.msc test TOP=..\sqlite There are several build options that can be set via the NMAKE command line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument diff --git a/manifest b/manifest index 3d90b0cede..7ebd38a5c1 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Mention\sthe\s".selftest"\scommand\sin\sthe\s".help"\soutput\sof\sthe\sCLI. -D 2017-03-15T14:20:34.151 +C Add\sthe\s"Obtaining\sThe\sCode"\ssubsection\sin\sthe\stop-level\sREADME.md\sfile.\nNo\schanges\sto\scode. +D 2017-03-15T19:11:29.617 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a -F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 +F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9 F VERSION 3605fa447e4623f5ff4a6adc97b1fde9a257b8f2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -1564,7 +1564,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bcf6bb08d8b07d3c4567bcd367f8d4011ce8baad28c25d4e8870bdf347dd48bd -R 3a48c86458783a20973121829bd123da +P 37f766dbad1f99ff86dd1b771bf443036e928e5b4d8abe55bbe4acf3362c7be2 +R 2482a0aa4c5093969a8d8859e4856dc3 U drh -Z 148bc384ce73e7778d1614d5fa3b82a8 +Z 10bbbdac78940d7fca566277c18da714 diff --git a/manifest.uuid b/manifest.uuid index 3ea504fab7..72a98aae6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37f766dbad1f99ff86dd1b771bf443036e928e5b4d8abe55bbe4acf3362c7be2 \ No newline at end of file +b1b1aa8b69aa80c83aec3380565f0b4ec0b6a6e033537becee098872da362e9a \ No newline at end of file From 6f25936c9c12c93a95aad4277964a82b2f18b07a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Mar 2017 20:27:46 +0000 Subject: [PATCH 251/292] Updates to README files under the ext/ hierarchy. No changes to code. FossilOrigin-Name: 029bc5d224bcbdcca2307710539b133c39e2a27b971c28b294a1f517b80cb418 --- ext/README.md | 8 ++++++++ ext/README.txt | 2 -- ext/misc/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ manifest | 13 +++++++------ manifest.uuid | 2 +- 5 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 ext/README.md delete mode 100644 ext/README.txt create mode 100644 ext/misc/README.md diff --git a/ext/README.md b/ext/README.md new file mode 100644 index 0000000000..933a33d053 --- /dev/null +++ b/ext/README.md @@ -0,0 +1,8 @@ +## Loadable Extensions + +Various [loadable extensions](https://www.sqlite.org/loadext.html) for +SQLite are found in subfolders. + +Most subfolders are dedicated to a single loadable extension (for +example FTS5, or RTREE). But the misc/ subfolder contains a collection +of smaller single-file extensions. diff --git a/ext/README.txt b/ext/README.txt deleted file mode 100644 index 009495f59d..0000000000 --- a/ext/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -Version loadable extensions to SQLite are found in subfolders -of this folder. diff --git a/ext/misc/README.md b/ext/misc/README.md new file mode 100644 index 0000000000..970a09c2e9 --- /dev/null +++ b/ext/misc/README.md @@ -0,0 +1,40 @@ +## Miscellaneous Extensions + +This folder contains a collection of smaller loadable extensions. +See for instructions on how +to compile and use loadable extensions. +Each extension in this folder is implemented in a single file of C code. + +Each source file contains a description in its header comment. See the +header comments for details about each extension. Additional notes are +as follows: + + * **carray.c** — This module implements the + [carray](https://www.sqlite.org/carray.html) table-valued function. + It is a good example of how to go about implementing a custom + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). + + * **dbdump.c** — This is not actually a loadable extension, but + rather a library that implements an approximate equivalent to the + ".dump" command of the + [command-line shell](https://www.sqlite.org/cli.html). + + * **memvfs.c** — This file implements a custom + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database + file in a single block of RAM. It serves as a good example of how + to implement a simple custom VFS. + + * **rot13.c** — This file implements the very simple rot13() + substitution function. This file makes a good template for implementing + new custom SQL functions for SQLite. + + * **series.c** — This is an implementation of the + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + It can make a good template for new custom virtual table implementations. + + * **shathree.c** — An implementation of the sha3() and + sha3_query() SQL functions. The file is named "shathree.c" instead + of "sha3.c" because the default entry point names in SQLite are based + on the source filename with digits removed, so if we used the name + "sha3.c" then the entry point would conflict with the prior "sha1.c" + extension. diff --git a/manifest b/manifest index 7ebd38a5c1..83da720fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"Obtaining\sThe\sCode"\ssubsection\sin\sthe\stop-level\sREADME.md\sfile.\nNo\schanges\sto\scode. -D 2017-03-15T19:11:29.617 +C Updates\sto\sREADME\sfiles\sunder\sthe\sext/\shierarchy.\s\sNo\schanges\sto\scode. +D 2017-03-15T20:27:46.132 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -36,7 +36,7 @@ F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 +F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef @@ -206,6 +206,7 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 @@ -1564,7 +1565,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 37f766dbad1f99ff86dd1b771bf443036e928e5b4d8abe55bbe4acf3362c7be2 -R 2482a0aa4c5093969a8d8859e4856dc3 +P b1b1aa8b69aa80c83aec3380565f0b4ec0b6a6e033537becee098872da362e9a +R ed52dc66e6f47f5c1fac043513c8c66c U drh -Z 10bbbdac78940d7fca566277c18da714 +Z 7b77c481896e5fef2abe165dd1f4d320 diff --git a/manifest.uuid b/manifest.uuid index 72a98aae6c..b973ec5b0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1b1aa8b69aa80c83aec3380565f0b4ec0b6a6e033537becee098872da362e9a \ No newline at end of file +029bc5d224bcbdcca2307710539b133c39e2a27b971c28b294a1f517b80cb418 \ No newline at end of file From 116b56a2ccb5671bd6ff668ad3aec104d8b4b9a9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Mar 2017 12:11:07 +0000 Subject: [PATCH 252/292] Fix a crash that could follow an OOM condition in the instr() SQL function. FossilOrigin-Name: 6e59e903e4e956617bddef0b94e5cae02d724ac8145940b57ab5b0f628759736 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/func.c | 6 ++++-- test/mallocM.test | 18 +++++++++++++++++- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 83da720fa0..e9aad067d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sREADME\sfiles\sunder\sthe\sext/\shierarchy.\s\sNo\schanges\sto\scode. -D 2017-03-15T20:27:46.132 +C Fix\sa\scrash\sthat\scould\sfollow\san\sOOM\scondition\sin\sthe\sinstr()\sSQL\sfunction. +D 2017-03-16T12:11:07.597 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -36,7 +36,7 @@ F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt +F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef @@ -356,7 +356,7 @@ F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c f12a581f342a6fd85d14c31e4fb84f16b3dd107f54d7728dddb62cebc79d7ce1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae -F src/func.c c67273e1ec08abbdcc14c189892a3ff6eeece86b +F src/func.c 72ed1518f59951daca3b3480331006f074041b4753ab652b46bbdaedb77f6d6c F src/global.c 4a34512d82fc5aa13c802db06bcfff5e1d3de955 F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -955,7 +955,7 @@ F test/mallocI.test 6c23a71df077fa5d387be90e7e669c5b368ca38a F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 27cb5566a6e5f2d76f9d4aa2eca45524401fd61e F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc -F test/mallocM.test 491001d1e273233048d265ec6d38fdd23745b0284f0c93bc98c94b64451c9c28 +F test/mallocM.test 78bbe9d3da84a5c679123cdb40d7b2010b18fc46e13897e4f253c6ba6fbff134 F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -1565,7 +1565,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b1b1aa8b69aa80c83aec3380565f0b4ec0b6a6e033537becee098872da362e9a -R ed52dc66e6f47f5c1fac043513c8c66c -U drh -Z 7b77c481896e5fef2abe165dd1f4d320 +P 029bc5d224bcbdcca2307710539b133c39e2a27b971c28b294a1f517b80cb418 +R e125f692ae2203427c312740eb052007 +U dan +Z 43eef8b6f1a791e34b2b986222f188cc diff --git a/manifest.uuid b/manifest.uuid index b973ec5b0b..808a7701e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -029bc5d224bcbdcca2307710539b133c39e2a27b971c28b294a1f517b80cb418 \ No newline at end of file +6e59e903e4e956617bddef0b94e5cae02d724ac8145940b57ab5b0f628759736 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 885725bc6b..181032a5e3 100644 --- a/src/func.c +++ b/src/func.c @@ -204,9 +204,11 @@ static void instrFunc( if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ zHaystack = sqlite3_value_blob(argv[0]); zNeedle = sqlite3_value_blob(argv[1]); - assert( zNeedle!=0 ); - assert( zHaystack!=0 || nHaystack==0 ); isText = 0; + /* The following condition may be true if the arguments passed to this + ** function are values returned by zeroblob() or similar and an OOM + ** occurs while expanding the blob value. */ + if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; }else{ zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); diff --git a/test/mallocM.test b/test/mallocM.test index 85a38acf32..4da3a9e112 100644 --- a/test/mallocM.test +++ b/test/mallocM.test @@ -21,7 +21,7 @@ sqlite3_db_config_lookaside db 0 0 0 do_execsql_test 1.0 { CREATE TABLE t1(x); } -do_faultsim_test 1 -faults oom-t* -body { +do_faultsim_test 1 -faults oom* -body { execsql { SELECT 'abc' FROM ( SELECT 'xyz' FROM t1 WHERE (SELECT 1) ) } @@ -29,4 +29,20 @@ do_faultsim_test 1 -faults oom-t* -body { faultsim_test_result {0 {}} } +do_execsql_test 2.0.1 { SELECT instr(x'', x'') } {1} +do_execsql_test 2.0.2 { SELECT instr(x'12345678', x'') } {1} +do_execsql_test 2.0.3 { SELECT instr(x'', x'1234') } {0} + +do_faultsim_test 2.1 -faults oom* -body { + execsql { SELECT instr (x'00', zeroblob(1)) } +} -test { + faultsim_test_result {0 1} +} + +do_faultsim_test 2.2 -faults oom* -body { + execsql { SELECT instr (zeroblob(1), x'00') } +} -test { + faultsim_test_result {0 1} +} + finish_test From 970ea372c41bf3fda4891740d74130a490809288 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Mar 2017 13:14:03 +0000 Subject: [PATCH 253/292] Fix a problem in the "showdb" utility that prevents it from correctly decoding cells with no content beyond the record header. FossilOrigin-Name: eb7680a1c047b2a33d6a0c9733fafaee11272377c627af10bbd541b6b7ed952b --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/showdb.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e9aad067d4..e0d90caf8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\sfollow\san\sOOM\scondition\sin\sthe\sinstr()\sSQL\sfunction. -D 2017-03-16T12:11:07.597 +C Fix\sa\sproblem\sin\sthe\s"showdb"\sutility\sthat\sprevents\sit\sfrom\scorrectly\ndecoding\scells\swith\sno\scontent\sbeyond\sthe\srecord\sheader. +D 2017-03-16T13:14:03.665 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1518,7 +1518,7 @@ F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c c695a5d5c8110640e0d9fadf5e254da90c79c36e +F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9 @@ -1565,7 +1565,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 029bc5d224bcbdcca2307710539b133c39e2a27b971c28b294a1f517b80cb418 -R e125f692ae2203427c312740eb052007 -U dan -Z 43eef8b6f1a791e34b2b986222f188cc +P 6e59e903e4e956617bddef0b94e5cae02d724ac8145940b57ab5b0f628759736 +R e2b1419a7c2797d673769ba867cc93d0 +U drh +Z 979d31964af3fe7c6823fdbe1eb4300f diff --git a/manifest.uuid b/manifest.uuid index 808a7701e3..8782dfa202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6e59e903e4e956617bddef0b94e5cae02d724ac8145940b57ab5b0f628759736 \ No newline at end of file +eb7680a1c047b2a33d6a0c9733fafaee11272377c627af10bbd541b6b7ed952b \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index d51a2fd83d..ba7a362258 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -535,7 +535,7 @@ static void decodeCell( j = i; nCol = 0; k = nHdr; - while( x+j Date: Thu, 16 Mar 2017 13:30:58 +0000 Subject: [PATCH 254/292] Fix an uninitialized variable reference in the text generator for "PRAGMA vdbe_trace=ON" output. Problem discovered by OSS-Fuzz. FossilOrigin-Name: e3d487162d1596ce125644f754ed9531ef4412f31f6837c3e31b7542b90602fe --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 8 ++------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index e0d90caf8e..94d43a58ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\s"showdb"\sutility\sthat\sprevents\sit\sfrom\scorrectly\ndecoding\scells\swith\sno\scontent\sbeyond\sthe\srecord\sheader. -D 2017-03-16T13:14:03.665 +C Fix\san\suninitialized\svariable\sreference\sin\sthe\stext\sgenerator\s\nfor\s"PRAGMA\svdbe_trace=ON"\soutput.\nProblem\sdiscovered\sby\sOSS-Fuzz. +D 2017-03-16T13:30:58.522 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -467,7 +467,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c f520378e510fd36bbf289921798dbc8f2b3dc30d +F src/vdbe.c 0e6e5e01aaa57f6d1784b7d6b6a352153e9d2c2ff395c67ca59aef54b8268cf6 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 @@ -1565,7 +1565,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6e59e903e4e956617bddef0b94e5cae02d724ac8145940b57ab5b0f628759736 -R e2b1419a7c2797d673769ba867cc93d0 +P eb7680a1c047b2a33d6a0c9733fafaee11272377c627af10bbd541b6b7ed952b +R 698ee80038f5639a9fb93e688b92867d U drh -Z 979d31964af3fe7c6823fdbe1eb4300f +Z 1c91cacf813bcc1311ca42c9e29c59cd diff --git a/manifest.uuid b/manifest.uuid index 8782dfa202..5411cb37d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb7680a1c047b2a33d6a0c9733fafaee11272377c627af10bbd541b6b7ed952b \ No newline at end of file +e3d487162d1596ce125644f754ed9531ef4412f31f6837c3e31b7542b90602fe \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5901ba0eb5..20bf09e3a4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -403,9 +403,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ }else{ c = 's'; } - - sqlite3_snprintf(100, zCsr, "%c", c); - zCsr += sqlite3Strlen30(zCsr); + *(zCsr++) = c; sqlite3_snprintf(100, zCsr, "%d[", pMem->n); zCsr += sqlite3Strlen30(zCsr); for(i=0; i<16 && in; i++){ @@ -417,9 +415,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ if( z<32 || z>126 ) *zCsr++ = '.'; else *zCsr++ = z; } - - sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); - zCsr += sqlite3Strlen30(zCsr); + *(zCsr++) = ']'; if( f & MEM_Zero ){ sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); zCsr += sqlite3Strlen30(zCsr); From b30574bcae87dd4efe6de37d47a215e112824fe2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Mar 2017 14:28:52 +0000 Subject: [PATCH 255/292] Simplified OOM detection in the instr() SQL function. FossilOrigin-Name: 6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 94d43a58ec..7d0466ed61 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\suninitialized\svariable\sreference\sin\sthe\stext\sgenerator\s\nfor\s"PRAGMA\svdbe_trace=ON"\soutput.\nProblem\sdiscovered\sby\sOSS-Fuzz. -D 2017-03-16T13:30:58.522 +C Simplified\sOOM\sdetection\sin\sthe\sinstr()\sSQL\sfunction. +D 2017-03-16T14:28:52.423 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -356,7 +356,7 @@ F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c f12a581f342a6fd85d14c31e4fb84f16b3dd107f54d7728dddb62cebc79d7ce1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae -F src/func.c 72ed1518f59951daca3b3480331006f074041b4753ab652b46bbdaedb77f6d6c +F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174 F src/global.c 4a34512d82fc5aa13c802db06bcfff5e1d3de955 F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -1565,7 +1565,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb7680a1c047b2a33d6a0c9733fafaee11272377c627af10bbd541b6b7ed952b -R 698ee80038f5639a9fb93e688b92867d +P e3d487162d1596ce125644f754ed9531ef4412f31f6837c3e31b7542b90602fe +R 6130240157d02268edef1462c36002d9 U drh -Z 1c91cacf813bcc1311ca42c9e29c59cd +Z 5ff31e1a870d72447c943f18c5a0c832 diff --git a/manifest.uuid b/manifest.uuid index 5411cb37d9..eee2f53d07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3d487162d1596ce125644f754ed9531ef4412f31f6837c3e31b7542b90602fe \ No newline at end of file +6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f \ No newline at end of file diff --git a/src/func.c b/src/func.c index 181032a5e3..3d5a059a9f 100644 --- a/src/func.c +++ b/src/func.c @@ -205,16 +205,12 @@ static void instrFunc( zHaystack = sqlite3_value_blob(argv[0]); zNeedle = sqlite3_value_blob(argv[1]); isText = 0; - /* The following condition may be true if the arguments passed to this - ** function are values returned by zeroblob() or similar and an OOM - ** occurs while expanding the blob value. */ - if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; }else{ zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; - if( zHaystack==0 || zNeedle==0 ) return; } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ N++; do{ From f5da7dbb0773facb0a6a12e25f4139a7c5087c74 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Mar 2017 18:14:39 +0000 Subject: [PATCH 256/292] If the user has not set it explicitly, set the "PRAGMA synchronous" setting to SQLITE_DEFAULT_SYNCHRONOUS when a database connection changes from wal to rollback journal mode. FossilOrigin-Name: 78030c0f52aa39fb2ab32c75c56e6bcefe6382b8df28b1909e3c911e42dbeca3 --- manifest | 19 +++--- manifest.uuid | 2 +- src/btree.c | 42 ++++++++----- src/ctime.c | 6 ++ src/test_config.c | 2 + test/sync2.test | 153 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 24 deletions(-) create mode 100644 test/sync2.test diff --git a/manifest b/manifest index 7d0466ed61..4d61ee44cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplified\sOOM\sdetection\sin\sthe\sinstr()\sSQL\sfunction. -D 2017-03-16T14:28:52.423 +C If\sthe\suser\shas\snot\sset\sit\sexplicitly,\sset\sthe\s"PRAGMA\ssynchronous"\ssetting\sto\nSQLITE_DEFAULT_SYNCHRONOUS\swhen\sa\sdatabase\sconnection\schanges\sfrom\swal\sto\nrollback\sjournal\smode. +D 2017-03-16T18:14:39.798 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -343,13 +343,13 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c e2bae0a03f73d119910fb35c9550987564065137 +F src/btree.c ace24955d01800acec28c79b038e1c291370f48a8955e8347cbd5339e39935de F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 43f903c9082040ced2b421543cb0300c2973647d F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 +F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c @@ -425,7 +425,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 F src/test_blob.c f65ac717da2618691cf9dad094e6da0219dcd208 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c 83179ea845479b5be9a651d014649e3f2722a1fe +F src/test_config.c edcba290248dc18736dd814c9b95863c6762e0b35753048d8cbe5bf65f7abfbb F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 @@ -1165,6 +1165,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 +F test/sync2.test 5ecf56be4c187a191253a1ec0786fe2df975dbf3dc0f0140cdf5d3690c90651c F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 @@ -1565,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3d487162d1596ce125644f754ed9531ef4412f31f6837c3e31b7542b90602fe -R 6130240157d02268edef1462c36002d9 -U drh -Z 5ff31e1a870d72447c943f18c5a0c832 +P 6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f +R 549202c1b901efe7d78e47bd0d1adc7f +U dan +Z 959d77178fc349b2ba7589eca45f1aa2 diff --git a/manifest.uuid b/manifest.uuid index eee2f53d07..1973398254 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f \ No newline at end of file +78030c0f52aa39fb2ab32c75c56e6bcefe6382b8df28b1909e3c911e42dbeca3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b99cab89ec..47bb460b0c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2861,6 +2861,31 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ #endif } +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt) +#endif /* ** Get a reference to pPage1 of the database file. This will @@ -2934,26 +2959,15 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ -#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS - sqlite3 *db; - Db *pDb; - if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ - while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 - ){ - pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; - sqlite3PagerSetFlags(pBt->pPager, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); - } - } -#endif + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); if( isOpen==0 ){ releasePage(pPage1); return SQLITE_OK; } } rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); } #endif diff --git a/src/ctime.c b/src/ctime.c index 3db3192095..20e94edba5 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -66,6 +66,12 @@ static const char * const azCompileOpt[] = { #if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif +#if SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#if SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif #if SQLITE_DIRECT_OVERFLOW_READ "DIRECT_OVERFLOW_READ", #endif diff --git a/src/test_config.c b/src/test_config.c index f409b1ca6f..a25b0b3525 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -740,6 +740,8 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); LINKVAR( DEFAULT_CACHE_SIZE ); LINKVAR( DEFAULT_PAGE_SIZE ); LINKVAR( DEFAULT_FILE_FORMAT ); + LINKVAR( DEFAULT_SYNCHRONOUS ); + LINKVAR( DEFAULT_WAL_SYNCHRONOUS ); LINKVAR( MAX_ATTACHED ); LINKVAR( MAX_DEFAULT_PAGE_SIZE ); LINKVAR( MAX_WORKER_THREADS ); diff --git a/test/sync2.test b/test/sync2.test new file mode 100644 index 0000000000..5f9c9aba93 --- /dev/null +++ b/test/sync2.test @@ -0,0 +1,153 @@ +# 2017 March 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specificly, it tests that "PRAGMA synchronous" appears to work. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sync2 + +# +# These tests are only applicable when pager pragma are +# enabled. Also, since every test uses an ATTACHed database, they +# are only run when ATTACH is enabled. +# +ifcapable !pager_pragmas||!attach { + finish_test + return +} + +proc execsql_sync {sql} { + set s $::sqlite_sync_count + set res [execsql $sql] + concat [expr $::sqlite_sync_count-$s] $res +} + +proc do_execsql_sync_test {tn sql res} { + uplevel [list do_test $tn [list execsql_sync $sql] [list {*}$res]] +} + +#----------------------------------------------------------------------- +# Tests for journal mode. +# +sqlite3 db test.db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} + +do_execsql_sync_test 1.1 { INSERT INTO t1 VALUES(3, 4) } 4 + +# synchronous=normal. So, 1 sync on the directory, 1 on the journal, 1 +# on the db file. 3 in total. +do_execsql_test 1.2.1 { PRAGMA main.synchronous = NORMAL } +do_execsql_test 1.2.2 { PRAGMA main.synchronous } 1 +do_execsql_sync_test 1.2.3 { INSERT INTO t1 VALUES(5, 6) } 3 + +# synchronous=off. No syncs. +do_execsql_test 1.3.1 { PRAGMA main.synchronous = OFF } +do_execsql_test 1.3.2 { PRAGMA main.synchronous } 0 +do_execsql_sync_test 1.3.3 { INSERT INTO t1 VALUES(7, 8) } 0 + +# synchronous=full, journal_mode=delete. So, 1 sync on the directory, +# 2 on the journal, 1 on the db file. 4 in total. +do_execsql_test 1.4.1 { PRAGMA main.synchronous = FULL } +do_execsql_test 1.4.2 { PRAGMA main.synchronous } 2 +do_execsql_sync_test 1.4.3 { INSERT INTO t1 VALUES(9, 10) } 4 + +#----------------------------------------------------------------------- +# Tests for wal mode. +# +do_execsql_test 1.5 { PRAGMA journal_mode = wal } {wal} + +# sync=full, journal_mode=wal. One sync on the directory, two on the +# wal file. +do_execsql_sync_test 1.6 { INSERT INTO t1 VALUES(11, 12) } 3 + +# One sync on the wal file. +do_execsql_sync_test 1.7 { INSERT INTO t1 VALUES(13, 14) } 1 + +# No syncs. +do_execsql_test 1.8.1 { PRAGMA main.synchronous = NORMAL } +do_execsql_test 1.8.2 { PRAGMA main.synchronous } 1 +do_execsql_sync_test 1.8.3 { INSERT INTO t1 VALUES(15, 16) } 0 + +# One sync on wal file, one on the db file. +do_execsql_sync_test 1.9 { PRAGMA wal_checkpoint } {2 0 3 3} + +# No syncs. +do_execsql_test 1.10.1 { PRAGMA main.synchronous = OFF } +do_execsql_test 1.10.2 { PRAGMA main.synchronous } 0 +do_execsql_sync_test 1.10.3 { INSERT INTO t1 VALUES(17, 18) } 0 + +#----------------------------------------------------------------------- +# Tests for the compile time settings SQLITE_DEFAULT_SYNCHRONOUS and +# SQLITE_DEFAULT_WAL_SYNCHRONOUS. These tests only run if the former +# is set to "2" and the latter to "1". This is not the default, but +# it is currently the recommended configuration. +# +# https://sqlite.org/compile.html#recommended_compile_time_options +# +if {$SQLITE_DEFAULT_SYNCHRONOUS==2 && $SQLITE_DEFAULT_WAL_SYNCHRONOUS==1} { + + db close + sqlite3 db test.db + + # Wal mode, sync=normal. The first transaction does one sync on directory, + # one on the wal file. The second does no syncs. + do_execsql_sync_test 1.11.1 { INSERT INTO t1 VALUES(19, 20) } 2 + do_execsql_sync_test 1.11.2 { INSERT INTO t1 VALUES(21, 22) } 0 + do_execsql_test 1.11.3 { PRAGMA main.synchronous } 1 + + # One sync on wal file, one on the db file. + do_execsql_sync_test 1.12 { PRAGMA wal_checkpoint } {2 0 2 2} + + # First transaction syncs the wal file once, the second not at all. + # one on the wal file. The second does no syncs. + do_execsql_sync_test 1.13.1 { INSERT INTO t1 VALUES(22, 23) } 1 + do_execsql_sync_test 1.13.2 { INSERT INTO t1 VALUES(24, 25) } 0 + + do_execsql_test 1.14 { PRAGMA journal_mode = delete } {delete} + + # Delete mode, sync=full. The first transaction does one sync on + # directory, two on the journal file, one on the db. The second does + # the same. + do_execsql_sync_test 1.15.1 { INSERT INTO t1 VALUES(26, 27) } 4 + do_execsql_sync_test 1.15.2 { INSERT INTO t1 VALUES(28, 29) } 4 + do_execsql_test 1.15.3 { PRAGMA main.synchronous } 2 + + # Switch back to wal mode. + do_execsql_test 1.16 { PRAGMA journal_mode = wal } {wal} + + do_execsql_sync_test 1.17.1 { INSERT INTO t1 VALUES(30, 31) } 2 + do_execsql_sync_test 1.17.2 { INSERT INTO t1 VALUES(32, 33) } 0 + do_execsql_test 1.17.3 { PRAGMA main.synchronous } 1 + + # Now set synchronous=off, then switch back to delete mode. Check + # that the db handle is still using synchronous=off. + do_execsql_test 1.18.3 { PRAGMA main.synchronous=off } + do_execsql_test 1.18 { PRAGMA journal_mode = delete } {delete} + + do_execsql_sync_test 1.19.1 { INSERT INTO t1 VALUES(34, 35) } 0 + do_execsql_sync_test 1.19.2 { INSERT INTO t1 VALUES(36, 37) } 0 + do_execsql_test 1.19.3 { PRAGMA main.synchronous } 0 + + # Close and reopen the db. Back to synchronous=normal. + db close + sqlite3 db test.db + do_execsql_sync_test 1.20.1 { INSERT INTO t1 VALUES(38, 39) } 4 + do_execsql_sync_test 1.20.2 { INSERT INTO t1 VALUES(40, 41) } 4 + do_execsql_test 1.20.3 { PRAGMA main.synchronous } 2 +} + +finish_test From fc8f4b658ccf5ee626b21115f336e6068623827d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Mar 2017 18:54:42 +0000 Subject: [PATCH 257/292] Fix a typo causing the build to fail if SQLITE_DEFAULT_SYNCHRONOUS==SQLITE_DEFAULT_WAL_SYNCHRONOUS. FossilOrigin-Name: df39adeaa4d73a3d4ec831843b8ec7fd449f8b2137bdc7d7c320958d00e98db6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4d61ee44cb..27d666d7ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\shas\snot\sset\sit\sexplicitly,\sset\sthe\s"PRAGMA\ssynchronous"\ssetting\sto\nSQLITE_DEFAULT_SYNCHRONOUS\swhen\sa\sdatabase\sconnection\schanges\sfrom\swal\sto\nrollback\sjournal\smode. -D 2017-03-16T18:14:39.798 +C Fix\sa\stypo\scausing\sthe\sbuild\sto\sfail\sif\sSQLITE_DEFAULT_SYNCHRONOUS==SQLITE_DEFAULT_WAL_SYNCHRONOUS. +D 2017-03-16T18:54:42.504 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -343,7 +343,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c ace24955d01800acec28c79b038e1c291370f48a8955e8347cbd5339e39935de +F src/btree.c ae0e0397e6ad58465bbf932239ee7539ca22f257c97b16c9d0960a1f5de743a3 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 43f903c9082040ced2b421543cb0300c2973647d @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f -R 549202c1b901efe7d78e47bd0d1adc7f +P 78030c0f52aa39fb2ab32c75c56e6bcefe6382b8df28b1909e3c911e42dbeca3 +R ce97b08e421fd719736eea9dd091c75c U dan -Z 959d77178fc349b2ba7589eca45f1aa2 +Z dc5f7c52928823522e8c3e89a1ed9d18 diff --git a/manifest.uuid b/manifest.uuid index 1973398254..57ec040514 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78030c0f52aa39fb2ab32c75c56e6bcefe6382b8df28b1909e3c911e42dbeca3 \ No newline at end of file +df39adeaa4d73a3d4ec831843b8ec7fd449f8b2137bdc7d7c320958d00e98db6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 47bb460b0c..52c0c47dd5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2884,7 +2884,7 @@ static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ } } #else -# define setDefaultSyncFlag(pBt) +# define setDefaultSyncFlag(pBt,safety_level) #endif /* From a3b2da9889ceb5ca041ca0eb2bc8d35190aae94c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Mar 2017 03:21:14 +0000 Subject: [PATCH 258/292] Fix a problem in the enhanced PRAGMA integrity_check where it verifies CHECK constraints: Do not be confused by the reuse of the Table.pCheck field by VIEWs with named columns. Problem discovered by OSS-Fuzz. FossilOrigin-Name: 019dd3d5ba4a596c4ec3b5f0de8402c72196af0faca9138edbc0f1f4957cae60 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/pragma.c | 1 + test/check.test | 11 ++++++++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index dde0392581..99c6df185c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\shas\snot\sset\sit\sexplicitly,\sset\sthe\s"PRAGMA\ssynchronous"\ssetting\sto\nSQLITE_DEFAULT_SYNCHRONOUS\swhen\sa\sdatabase\sconnection\schanges\sfrom\swal\sto\nrollback\sjournal\smode. -D 2017-03-16T18:55:21.135 +C Fix\sa\sproblem\sin\sthe\senhanced\sPRAGMA\sintegrity_check\swhere\sit\sverifies\nCHECK\sconstraints:\sDo\snot\sbe\sconfused\sby\sthe\sreuse\sof\sthe\sTable.pCheck\sfield\nby\sVIEWs\swith\snamed\scolumns.\s\sProblem\sdiscovered\sby\sOSS-Fuzz. +D 2017-03-17T03:21:14.865 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -393,7 +393,7 @@ F src/parse.y 48b03113704ee8bd78ee6996d81de7fbee22e105 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c bfaa7e5cbfc8eacaa0f3611d5ec2dca1339d6d301f1e9b429b49ca460d794a60 +F src/pragma.c 2b244434e76c7075edbcfd9e4d634899af0944ff01183b126d4671f7407c2368 F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -582,7 +582,7 @@ F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef -F test/check.test 92b23a91fb7be12fba7ee9ce518217e2919a21da +F test/check.test fb823c1aee9d5187b4a474f1728bb65f2a77303255e294e250a2328ab7e72832 F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1566,8 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d85eb5736781b43aa674d9544c7523b849b4e634f371702f8764b33e22e1e9f df39adeaa4d73a3d4ec831843b8ec7fd449f8b2137bdc7d7c320958d00e98db6 -R ce97b08e421fd719736eea9dd091c75c -T +closed df39adeaa4d73a3d4ec831843b8ec7fd449f8b2137bdc7d7c320958d00e98db6 -U dan -Z 3997e48cc61ec2c438e101b7960cfa80 +P 5c604479fda06714500959e121c719d1b6c8e54d1658eb9f560a4af95b7c5829 +R 5046d28129e3bd81658af4ffa15e3f46 +U drh +Z bf308f7cb4f4a256732c0018ceafc10b diff --git a/manifest.uuid b/manifest.uuid index 195d935585..3d54c9e4c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c604479fda06714500959e121c719d1b6c8e54d1658eb9f560a4af95b7c5829 \ No newline at end of file +019dd3d5ba4a596c4ec3b5f0de8402c72196af0faca9138edbc0f1f4957cae60 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 488a1d194c..6c27fdc5c9 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1505,6 +1505,7 @@ void sqlite3Pragma( int iDataCur, iIdxCur; int r1 = -1; + if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ if( pTab->pCheck==0 && (pTab->tabFlags & TF_HasNotNull)==0 && (pTab->pIndex==0 || isQuick) diff --git a/test/check.test b/test/check.test index 19f252677c..2100aebb85 100644 --- a/test/check.test +++ b/test/check.test @@ -481,6 +481,15 @@ do_catchsql_test 9.3 { UPDATE t1 SET c=a*2 WHERE a=1; } {1 {CHECK constraint failed: c-check}} - +# Integrity check on a VIEW with columns. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test 10.1 { + CREATE TABLE t1(x); + CREATE VIEW v1(y) AS SELECT x FROM t1; + PRAGMA integrity_check; +} {ok} finish_test From 1f9144ed41cfd73d32347b3be47cbca2e8c3fc4e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Mar 2017 13:59:06 +0000 Subject: [PATCH 259/292] Fix a buffer overread in debugging routine sqlite3VdbeMemPrettyPrint(). Problem discovered by OSS-Fuzz. FossilOrigin-Name: f336fba7d7d41b91a5000d01dddf785821fa79ea31dbd8d1f769d55f7e871896 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 9 +++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 99c6df185c..77a9eebb02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\senhanced\sPRAGMA\sintegrity_check\swhere\sit\sverifies\nCHECK\sconstraints:\sDo\snot\sbe\sconfused\sby\sthe\sreuse\sof\sthe\sTable.pCheck\sfield\nby\sVIEWs\swith\snamed\scolumns.\s\sProblem\sdiscovered\sby\sOSS-Fuzz. -D 2017-03-17T03:21:14.865 +C Fix\sa\sbuffer\soverread\sin\sdebugging\sroutine\ssqlite3VdbeMemPrettyPrint().\nProblem\sdiscovered\sby\sOSS-Fuzz. +D 2017-03-17T13:59:06.018 F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -467,7 +467,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 0e6e5e01aaa57f6d1784b7d6b6a352153e9d2c2ff395c67ca59aef54b8268cf6 +F src/vdbe.c 3b4221473438a047d85839129c5e71eab80d8d19487c71e0ac6802fb330ae834 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c604479fda06714500959e121c719d1b6c8e54d1658eb9f560a4af95b7c5829 -R 5046d28129e3bd81658af4ffa15e3f46 -U drh -Z bf308f7cb4f4a256732c0018ceafc10b +P 019dd3d5ba4a596c4ec3b5f0de8402c72196af0faca9138edbc0f1f4957cae60 +R 6b67c5433a6804d9ab2b5f747ce1957d +U dan +Z cc6693cd745156921be8476a07d72767 diff --git a/manifest.uuid b/manifest.uuid index 3d54c9e4c6..7fc9e0264a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -019dd3d5ba4a596c4ec3b5f0de8402c72196af0faca9138edbc0f1f4957cae60 \ No newline at end of file +f336fba7d7d41b91a5000d01dddf785821fa79ea31dbd8d1f769d55f7e871896 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 20bf09e3a4..7eb2b6f6f7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2657,8 +2657,13 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. */ - static u8 aZero[8]; /* This is the bogus content */ + ** content from disk. + ** + ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the + ** buffer passed to it, debugging function VdbeMemPrettyPrint() may + ** read up to 16. So 16 bytes of bogus content is supplied. + */ + static u8 aZero[16]; /* This is the bogus content */ sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); From 59d705ab808645ac865ad14e19d47f688064005f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Mar 2017 14:15:06 +0000 Subject: [PATCH 260/292] Fix the Makefile.in so that it builds the ossshell test program correctly. FossilOrigin-Name: 36f5602ec9fb8e404c5250e18b1db877ac7bee643918b94afd51808134ea7900 --- Makefile.in | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile.in b/Makefile.in index 0b40cb458b..9076ffc085 100644 --- a/Makefile.in +++ b/Makefile.in @@ -621,7 +621,8 @@ fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(TLIBS) + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ + $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) diff --git a/manifest b/manifest index 77a9eebb02..6f14fa4cca 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Fix\sa\sbuffer\soverread\sin\sdebugging\sroutine\ssqlite3VdbeMemPrettyPrint().\nProblem\sdiscovered\sby\sOSS-Fuzz. -D 2017-03-17T13:59:06.018 -F Makefile.in 9605f4c49eace601d5c12c85dd6e037cc613a6d823e857614ba26b42f1285db0 +C Fix\sthe\sMakefile.in\sso\sthat\sit\sbuilds\sthe\sossshell\stest\sprogram\scorrectly. +D 2017-03-17T14:15:06.628 +F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 019dd3d5ba4a596c4ec3b5f0de8402c72196af0faca9138edbc0f1f4957cae60 -R 6b67c5433a6804d9ab2b5f747ce1957d -U dan -Z cc6693cd745156921be8476a07d72767 +P f336fba7d7d41b91a5000d01dddf785821fa79ea31dbd8d1f769d55f7e871896 +R a0acb13b6691b2f29cdfa3d14a75a6b3 +U drh +Z a464f97090f56a22de007bae0707b7f9 diff --git a/manifest.uuid b/manifest.uuid index 7fc9e0264a..cdf2f41fac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f336fba7d7d41b91a5000d01dddf785821fa79ea31dbd8d1f769d55f7e871896 \ No newline at end of file +36f5602ec9fb8e404c5250e18b1db877ac7bee643918b94afd51808134ea7900 \ No newline at end of file From f53524b4f72be7e7cf96fdec983000c9e4c5a85a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Mar 2017 14:59:40 +0000 Subject: [PATCH 261/292] Add the --show-errors and --show-max-delay command-line options to the ossshell test program. FossilOrigin-Name: 626bdca98e0cd78ae873d97e75bb7d544ca18759c9f1e67f4adf03daca7fe5bf --- manifest | 14 +++++------ manifest.uuid | 2 +- test/ossfuzz.c | 67 +++++++++++++++++++++++++++++++++++++++++-------- test/ossshell.c | 33 ++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 6f14fa4cca..24baa432d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sMakefile.in\sso\sthat\sit\sbuilds\sthe\sossshell\stest\sprogram\scorrectly. -D 2017-03-17T14:15:06.628 +C Add\sthe\s--show-errors\sand\s--show-max-delay\scommand-line\soptions\sto\sthe\nossshell\stest\sprogram. +D 2017-03-17T14:59:40.532 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1009,8 +1009,8 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c 6dc75478809cfbd4609409a87179ddc2ffaa092e8adb27c1982c5a944a7dd81f -F test/ossshell.c d9f1a6f43e7bab45d6be857a5800f5d4a1861db3 +F test/ossfuzz.c 756ca4bede67ec22e3a700b1168bad767dc6fc69ede414c4ab87cfcfcceb4075 +F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f336fba7d7d41b91a5000d01dddf785821fa79ea31dbd8d1f769d55f7e871896 -R a0acb13b6691b2f29cdfa3d14a75a6b3 +P 36f5602ec9fb8e404c5250e18b1db877ac7bee643918b94afd51808134ea7900 +R 78dfbe98cc4f1c1387677674bd1f6a34 U drh -Z a464f97090f56a22de007bae0707b7f9 +Z 8d96b967fb72fb140eaa51fb70892dfa diff --git a/manifest.uuid b/manifest.uuid index cdf2f41fac..6144ff8028 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36f5602ec9fb8e404c5250e18b1db877ac7bee643918b94afd51808134ea7900 \ No newline at end of file +626bdca98e0cd78ae873d97e75bb7d544ca18759c9f1e67f4adf03daca7fe5bf \ No newline at end of file diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 97d101e17a..6790d194d5 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -4,8 +4,26 @@ */ #include #include +#include +#include #include "sqlite3.h" +/* Global debugging settings. OSS-Fuzz will have all debugging turned +** off. But if LLVMFuzzerTestOneInput() is called interactively from +** the ossshell utility program, then these flags might be set. +*/ +static unsigned mDebug = 0; +#define FUZZ_SQL_TRACE 0x0001 /* Set an sqlite3_trace() callback */ +#define FUZZ_SHOW_MAX_DELAY 0x0002 /* Show maximum progress callback delay */ +#define FUZZ_SHOW_ERRORS 0x0004 /* Print error messages from SQLite */ + +/* The ossshell utility program invokes this interface to see the +** debugging flags. Unused by OSS-Fuzz. +*/ +void ossfuzz_set_debug_flags(unsigned x){ + mDebug = x; +} + /* Return the current real-world time in milliseconds since the ** Julian epoch (-4714-11-24). */ @@ -23,6 +41,17 @@ static sqlite3_int64 timeOfDay(void){ return t; } +/* An instance of the following object is passed by pointer as the +** client data to various callbacks. +*/ +typedef struct FuzzCtx { + sqlite3 *db; /* The database connection */ + sqlite3_int64 iCutoffTime; /* Stop processing at this time. */ + sqlite3_int64 iLastCb; /* Time recorded for previous progress callback */ + sqlite3_int64 mxInterval; /* Longest interval between two progress calls */ + unsigned nCb; /* Number of progress callbacks */ +} FuzzCtx; + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* ** Progress handler callback. @@ -30,9 +59,14 @@ static sqlite3_int64 timeOfDay(void){ ** The argument is the cutoff-time after which all processing should ** stop. So return non-zero if the cut-off time is exceeded. */ -static int progress_handler(void *pReturn) { - sqlite3_int64 iCutoffTime = *(sqlite3_int64*)pReturn; - return timeOfDay()>=iCutoffTime; +static int progress_handler(void *pClientData) { + FuzzCtx *p = (FuzzCtx*)pClientData; + sqlite3_int64 iNow = timeOfDay(); + int rc = iNow>=p->iCutoffTime; + sqlite3_int64 iDiff = iNow - p->iLastCb; + if( iDiff > p->mxInterval ) p->mxInterval = iDiff; + p->nCb++; + return rc; } #endif @@ -54,12 +88,12 @@ static int exec_handler(void *pCnt, int argc, char **argv, char **namev){ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { int execCnt = 0; /* Abort row callback when count reaches zero */ char *zErrMsg = 0; /* Error message returned by sqlite_exec() */ - sqlite3 *db; /* The database connection */ uint8_t uSelector; /* First byte of input data[] */ int rc; /* Return code from various interfaces */ char *zSql; /* Zero-terminated copy of data[] */ - sqlite3_int64 iCutoff; /* Cutoff timer */ + FuzzCtx cx; /* Fuzzing context */ + memset(&cx, 0, sizeof(cx)); if( size<3 ) return 0; /* Early out if unsufficient data */ /* Extract the selector byte from the beginning of the input. But only @@ -72,7 +106,7 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { } /* Open the database connection. Only use an in-memory database. */ - rc = sqlite3_open_v2("fuzz.db", &db, + rc = sqlite3_open_v2("fuzz.db", &cx.db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, 0); if( rc ) return 0; @@ -82,12 +116,13 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { ** (which will block further processing) if more than 10 seconds have ** elapsed since the start of the test. */ - iCutoff = timeOfDay() + 10000; /* Now + 10 seconds */ - sqlite3_progress_handler(db, 10, progress_handler, (void*)&iCutoff); + cx.iLastCb = timeOfDay(); + cx.iCutoffTime = cx.iLastCb + 10000; /* Now + 10 seconds */ + sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); #endif /* Bit 1 of the selector enables foreign key constraints */ - sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); + sqlite3_db_config(cx.db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); uSelector >>= 1; /* Remaining bits of the selector determine a limit on the number of @@ -97,11 +132,21 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* Run the SQL. The sqlite_exec() interface expects a zero-terminated ** string, so make a copy. */ zSql = sqlite3_mprintf("%.*s", (int)size, data); - sqlite3_exec(db, zSql, exec_handler, (void*)&execCnt, &zErrMsg); + sqlite3_exec(cx.db, zSql, exec_handler, (void*)&execCnt, &zErrMsg); + + /* Show any errors */ + if( (mDebug & FUZZ_SHOW_ERRORS)!=0 && zErrMsg ){ + printf("Error: %s\n", zErrMsg); + } /* Cleanup and return */ sqlite3_free(zErrMsg); sqlite3_free(zSql); - sqlite3_close(db); + sqlite3_close(cx.db); + + if( mDebug & FUZZ_SHOW_MAX_DELAY ){ + printf("Progress callback count....... %d\n", cx.nCb); + printf("Max time between callbacks.... %d ms\n", (int)cx.mxInterval); + } return 0; } diff --git a/test/ossshell.c b/test/ossshell.c index 15902a9122..00cc3391c8 100644 --- a/test/ossshell.c +++ b/test/ossshell.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "sqlite3.h" /* @@ -16,6 +17,13 @@ */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); +/* Must match equivalent #defines in ossfuzz.c */ +#define FUZZ_SQL_TRACE 0x0001 /* Set an sqlite3_trace() callback */ +#define FUZZ_SHOW_MAX_DELAY 0x0002 /* Show maximum progress callback delay */ +#define FUZZ_SHOW_ERRORS 0x0004 /* Show SQL errors */ +extern void ossfuzz_set_debug_flags(unsigned); + + /* ** Read files named on the command-line and invoke the fuzzer for @@ -27,9 +35,32 @@ int main(int argc, char **argv){ int nErr = 0; uint8_t *zBuf = 0; size_t sz; + unsigned mDebug = 0; for(i=1; i Date: Fri, 17 Mar 2017 22:50:16 +0000 Subject: [PATCH 262/292] Begin enforcing the SQLITE_LIMIT_VDBE_OP. The documentation warned that this day might come. FossilOrigin-Name: ef5914617088cbf89bfae88f63ea959a07f02dff387ddc2b43948ad99c6a97b8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite.h.in | 8 +++++--- src/sqliteLimit.h | 2 +- src/vdbeaux.c | 6 ++++++ 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 24baa432d0..db77af64f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--show-errors\sand\s--show-max-delay\scommand-line\soptions\sto\sthe\nossshell\stest\sprogram. -D 2017-03-17T14:59:40.532 +C Begin\senforcing\sthe\sSQLITE_LIMIT_VDBE_OP.\s\sThe\sdocumentation\swarned\sthat\sthis\nday\smight\scome. +D 2017-03-17T22:50:16.704 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -402,11 +402,11 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa F src/shell.c 77054c021069ec0b65d3d620aab031f97c59b4e42ac7c31544ea68933b581104 -F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 +F src/sqlite.h.in 5f72af637ba0f35c2b25f8f97ea63bac6f4ea133eb904ee0b6d61450401507d3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 -F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 +F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79 @@ -471,7 +471,7 @@ F src/vdbe.c 3b4221473438a047d85839129c5e71eab80d8d19487c71e0ac6802fb330ae834 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 -F src/vdbeaux.c 57361f2e761d92a254638bdbfc03fc68ae6aebc6 +F src/vdbeaux.c ecd0468611925d218e1eb4b3f538907904b136f0e15e333291a232b521bfcef1 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36f5602ec9fb8e404c5250e18b1db877ac7bee643918b94afd51808134ea7900 -R 78dfbe98cc4f1c1387677674bd1f6a34 +P 626bdca98e0cd78ae873d97e75bb7d544ca18759c9f1e67f4adf03daca7fe5bf +R 577cbf1218e05c6f890fd56d3bd8c3f1 U drh -Z 8d96b967fb72fb140eaa51fb70892dfa +Z db7430243d899e77772ca775a4ab8f59 diff --git a/manifest.uuid b/manifest.uuid index 6144ff8028..0f60fb4ebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -626bdca98e0cd78ae873d97e75bb7d544ca18759c9f1e67f4adf03daca7fe5bf \ No newline at end of file +ef5914617088cbf89bfae88f63ea959a07f02dff387ddc2b43948ad99c6a97b8 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 076a011319..5dd4419f43 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3424,9 +3424,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_VDBE_OP]] ^(
      SQLITE_LIMIT_VDBE_OP
      **
      The maximum number of instructions in a virtual machine program -** used to implement an SQL statement. This limit is not currently -** enforced, though that might be added in some future release of -** SQLite.
      )^ +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned. +** A value of 0 means "unlimited".)^ ** ** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
      SQLITE_LIMIT_FUNCTION_ARG
      **
      The maximum number of arguments on a function.
      )^ @@ -3464,6 +3465,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 + /* ** CAPI3REF: Compiling An SQL Statement ** KEYWORDS: {SQL statement compiler} diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 0554e61581..28e7a41cc3 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -87,7 +87,7 @@ ** Not currently enforced. */ #ifndef SQLITE_MAX_VDBE_OP -# define SQLITE_MAX_VDBE_OP 25000 +# define SQLITE_MAX_VDBE_OP 250000000 #endif /* diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ab4aad7a0f..a8f215420a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -117,6 +117,12 @@ static int growOpArray(Vdbe *v, int nOp){ UNUSED_PARAMETER(nOp); #endif + /* Ensure that the size of a VDBE does not grow too large */ + if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){ + sqlite3OomFault(p->db); + return SQLITE_NOMEM; + } + assert( nOp<=(1024/sizeof(Op)) ); assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); From 544cab7651100628ed290acc2a2fee394b7568aa Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Mar 2017 22:51:28 +0000 Subject: [PATCH 263/292] Set a reasonable limit on the number of opcodes in a prepared statement for ossfuzz.c. This should prevent timeouts in OSS-Fuzz when it generates totally unreasonable queries. FossilOrigin-Name: f74899ed2c78019abb406432a74dcd42a0ff8d9add005f8544dc4a8905f232eb --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/ossfuzz.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index db77af64f9..3396b9cc13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\senforcing\sthe\sSQLITE_LIMIT_VDBE_OP.\s\sThe\sdocumentation\swarned\sthat\sthis\nday\smight\scome. -D 2017-03-17T22:50:16.704 +C Set\sa\sreasonable\slimit\son\sthe\snumber\sof\sopcodes\sin\sa\sprepared\sstatement\nfor\sossfuzz.c.\s\sThis\sshould\sprevent\stimeouts\sin\sOSS-Fuzz\swhen\sit\sgenerates\ntotally\sunreasonable\squeries. +D 2017-03-17T22:51:28.665 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1009,7 +1009,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c 756ca4bede67ec22e3a700b1168bad767dc6fc69ede414c4ab87cfcfcceb4075 +F test/ossfuzz.c 8c4d62e156352ecb97eb868fcff20d828a279af67a1c1310779f63464d6f009f F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 626bdca98e0cd78ae873d97e75bb7d544ca18759c9f1e67f4adf03daca7fe5bf -R 577cbf1218e05c6f890fd56d3bd8c3f1 +P ef5914617088cbf89bfae88f63ea959a07f02dff387ddc2b43948ad99c6a97b8 +R 3a60b2fbebe94ced9367e16b751bb1ee U drh -Z db7430243d899e77772ca775a4ab8f59 +Z bc390a309335be50f7a4bf79ab7f90df diff --git a/manifest.uuid b/manifest.uuid index 0f60fb4ebb..e97c1e2b26 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef5914617088cbf89bfae88f63ea959a07f02dff387ddc2b43948ad99c6a97b8 \ No newline at end of file +f74899ed2c78019abb406432a74dcd42a0ff8d9add005f8544dc4a8905f232eb \ No newline at end of file diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 6790d194d5..0c2b1320e4 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -121,6 +121,9 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); #endif + /* Set a limit on the maximum size of a prepared statement */ + sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); + /* Bit 1 of the selector enables foreign key constraints */ sqlite3_db_config(cx.db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); uSelector >>= 1; From 46acfc2b4737258bf4af6f34f9ba88f0281d73bb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Mar 2017 23:08:11 +0000 Subject: [PATCH 264/292] Fix an error in the newly revised documentation for SQLITE_LIMIT_VDBE_OP. No changes to code. FossilOrigin-Name: f4cf8635e6fec6f04075cc067aaa71abc4f71739068e0ad2c44609dcb8691009 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3396b9cc13..0f47d77aec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sa\sreasonable\slimit\son\sthe\snumber\sof\sopcodes\sin\sa\sprepared\sstatement\nfor\sossfuzz.c.\s\sThis\sshould\sprevent\stimeouts\sin\sOSS-Fuzz\swhen\sit\sgenerates\ntotally\sunreasonable\squeries. -D 2017-03-17T22:51:28.665 +C Fix\san\serror\sin\sthe\snewly\srevised\sdocumentation\sfor\sSQLITE_LIMIT_VDBE_OP.\nNo\schanges\sto\scode. +D 2017-03-17T23:08:11.772 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -402,7 +402,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa F src/shell.c 77054c021069ec0b65d3d620aab031f97c59b4e42ac7c31544ea68933b581104 -F src/sqlite.h.in 5f72af637ba0f35c2b25f8f97ea63bac6f4ea133eb904ee0b6d61450401507d3 +F src/sqlite.h.in e992605a306ee4f58d2ecbbc665f71c0404a3c815fe90dd8e0c0e8f7b9b11166 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef5914617088cbf89bfae88f63ea959a07f02dff387ddc2b43948ad99c6a97b8 -R 3a60b2fbebe94ced9367e16b751bb1ee +P f74899ed2c78019abb406432a74dcd42a0ff8d9add005f8544dc4a8905f232eb +R 2c6af10e620c43369539bc210339bdfc U drh -Z bc390a309335be50f7a4bf79ab7f90df +Z 53bb8f51c26aa8f2582ebee07b281f74 diff --git a/manifest.uuid b/manifest.uuid index e97c1e2b26..5630824735 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f74899ed2c78019abb406432a74dcd42a0ff8d9add005f8544dc4a8905f232eb \ No newline at end of file +f4cf8635e6fec6f04075cc067aaa71abc4f71739068e0ad2c44609dcb8691009 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5dd4419f43..48e21e175b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3426,8 +3426,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); **
      The maximum number of instructions in a virtual machine program ** used to implement an SQL statement. If [sqlite3_prepare_v2()] or ** the equivalent tries to allocate space for more than this many opcodes -** in a single prepared statement, an SQLITE_NOMEM error is returned. -** A value of 0 means "unlimited".
      )^ +** in a single prepared statement, an SQLITE_NOMEM error is returned.)^ ** ** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
      SQLITE_LIMIT_FUNCTION_ARG
      **
      The maximum number of arguments on a function.
      )^ From a80e160634c95ab4d72ed0760d7a70eca458334a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Mar 2017 13:59:46 +0000 Subject: [PATCH 265/292] Only do the specialized MacOS single-core zone_malloc initialization if compiled with the SQLITE_MIGHT_BE_SINGLE_CORE flag. This avoids a (harmless) warning about OSAtomicCompareAndSwapPtrBarrier() being deprecated. FossilOrigin-Name: 4e6a03d9e12b120d15946b025f97a97697cb8e8af543c238ffda220c9e3f28f4 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/mem1.c | 59 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 0f47d77aec..d7127e3e78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\snewly\srevised\sdocumentation\sfor\sSQLITE_LIMIT_VDBE_OP.\nNo\schanges\sto\scode. -D 2017-03-17T23:08:11.772 +C Only\sdo\sthe\sspecialized\sMacOS\ssingle-core\szone_malloc\sinitialization\sif\ncompiled\swith\sthe\sSQLITE_MIGHT_BE_SINGLE_CORE\sflag.\s\sThis\savoids\sa\s(harmless)\nwarning\sabout\sOSAtomicCompareAndSwapPtrBarrier()\sbeing\sdeprecated. +D 2017-03-18T13:59:46.558 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -368,7 +368,7 @@ F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c fd7cd6fe21d46fe0a4186367dd8dc26d87b787eb +F src/mem1.c 79bf195f445bf7e66cadd121849837c3152fbd2f542326bbed3073b6902450c2 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f74899ed2c78019abb406432a74dcd42a0ff8d9add005f8544dc4a8905f232eb -R 2c6af10e620c43369539bc210339bdfc +P f4cf8635e6fec6f04075cc067aaa71abc4f71739068e0ad2c44609dcb8691009 +R 07e88bce7a34b04a1e3a36ad2d4b6f58 U drh -Z 53bb8f51c26aa8f2582ebee07b281f74 +Z 906adb059f738e34206863c58e96feb3 diff --git a/manifest.uuid b/manifest.uuid index 5630824735..4d805efd7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4cf8635e6fec6f04075cc067aaa71abc4f71739068e0ad2c44609dcb8691009 \ No newline at end of file +4e6a03d9e12b120d15946b025f97a97697cb8e8af543c238ffda220c9e3f28f4 \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index efc84c41d7..02ed59b4d4 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -57,7 +57,9 @@ */ #include #include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE #include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); @@ -236,33 +238,46 @@ static int sqlite3MemRoundup(int n){ */ static int sqlite3MemInit(void *NotUsed){ #if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) - int cpuCount; - size_t len; if( _sqliteZone_ ){ return SQLITE_OK; } - len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ - sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); - if( cpuCount>1 ){ - /* defer MT decisions to system malloc */ - _sqliteZone_ = malloc_default_zone(); - }else{ - /* only 1 core, use our own zone to contention over global locks, - ** e.g. we have our own dedicated locks */ - bool success; - malloc_zone_t* newzone = malloc_create_zone(4096, 0); - malloc_set_zone_name(newzone, "Sqlite_Heap"); - do{ - success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, - (void * volatile *)&_sqliteZone_); - }while(!_sqliteZone_); - if( !success ){ - /* somebody registered a zone first */ - malloc_destroy_zone(newzone); +#ifndef SQLITE_MIGHT_BE_SINGLE_CORE + /* If not compiled with the SQLITE_MIGHT_BE_SINGLE_CORE flag, assume + ** that multiple cores are always available. This is the default case. + */ + _sqliteZone_ = malloc_default_zone(); +#else + /* With the SQLITE_MIGHT_BE_SINGLE_CORE compile-time option, check the + ** number of cores. Different malloc() strategies are used for single-core and + ** multi-core machines. + */ + { + int cpuCount; + size_t len; + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + bool success; + malloc_zone_t* newzone = malloc_create_zone(4096, 0); + malloc_set_zone_name(newzone, "Sqlite_Heap"); + do{ + success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, + (void * volatile *)&_sqliteZone_); + }while(!_sqliteZone_); + if( !success ){ + /* somebody registered a zone first */ + malloc_destroy_zone(newzone); + } } } -#endif +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ UNUSED_PARAMETER(NotUsed); return SQLITE_OK; } From d3d52ef1471c670a1bb04c0dcf05cf7866c98cd7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 13:03:39 +0000 Subject: [PATCH 266/292] Documentation fix: SQLITE_SOURCE_ID is a now a SHA3-256 hash. FossilOrigin-Name: 2aa22509e7c8a1f09b16e4544c95d0b77503daed1f83106ccc18dee8bd9487a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d7127e3e78..5f21e2e7ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\sdo\sthe\sspecialized\sMacOS\ssingle-core\szone_malloc\sinitialization\sif\ncompiled\swith\sthe\sSQLITE_MIGHT_BE_SINGLE_CORE\sflag.\s\sThis\savoids\sa\s(harmless)\nwarning\sabout\sOSAtomicCompareAndSwapPtrBarrier()\sbeing\sdeprecated. -D 2017-03-18T13:59:46.558 +C Documentation\sfix:\s\sSQLITE_SOURCE_ID\sis\sa\snow\sa\sSHA3-256\shash. +D 2017-03-20T13:03:39.888 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -402,7 +402,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa F src/shell.c 77054c021069ec0b65d3d620aab031f97c59b4e42ac7c31544ea68933b581104 -F src/sqlite.h.in e992605a306ee4f58d2ecbbc665f71c0404a3c815fe90dd8e0c0e8f7b9b11166 +F src/sqlite.h.in 723107d97f2345a7c103632169dc61366121c4ab65d75a7d83c6dc0e5bbe5ca4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4cf8635e6fec6f04075cc067aaa71abc4f71739068e0ad2c44609dcb8691009 -R 07e88bce7a34b04a1e3a36ad2d4b6f58 +P 4e6a03d9e12b120d15946b025f97a97697cb8e8af543c238ffda220c9e3f28f4 +R 3e310c01c37c6ac62b74a767e914605e U drh -Z 906adb059f738e34206863c58e96feb3 +Z 061c837491d90260f4d8f777c55b50b5 diff --git a/manifest.uuid b/manifest.uuid index 4d805efd7a..611148833a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e6a03d9e12b120d15946b025f97a97697cb8e8af543c238ffda220c9e3f28f4 \ No newline at end of file +2aa22509e7c8a1f09b16e4544c95d0b77503daed1f83106ccc18dee8bd9487a4 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 48e21e175b..97294c3f81 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -114,8 +114,8 @@ extern "C" { ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID -** string contains the date and time of the check-in (UTC) and an SHA1 -** hash of the entire source tree. +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], From 978896e0eb0a27354e3a19c195718a9e2e7a13ca Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 14:44:07 +0000 Subject: [PATCH 267/292] Fix the check.test script so that it works on Windows. FossilOrigin-Name: 8822eb5d790bf8d4302c3f2dc05ab672193b309fb11771c4b3eea8a77f37e299 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/check.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5f21e2e7ce..ad0fdedb4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Documentation\sfix:\s\sSQLITE_SOURCE_ID\sis\sa\snow\sa\sSHA3-256\shash. -D 2017-03-20T13:03:39.888 +C Fix\sthe\scheck.test\sscript\sso\sthat\sit\sworks\son\sWindows. +D 2017-03-20T14:44:07.253 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -582,7 +582,7 @@ F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef -F test/check.test fb823c1aee9d5187b4a474f1728bb65f2a77303255e294e250a2328ab7e72832 +F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e6a03d9e12b120d15946b025f97a97697cb8e8af543c238ffda220c9e3f28f4 -R 3e310c01c37c6ac62b74a767e914605e +P 2aa22509e7c8a1f09b16e4544c95d0b77503daed1f83106ccc18dee8bd9487a4 +R 2a38f53db76a379174ef41cee348ccde U drh -Z 061c837491d90260f4d8f777c55b50b5 +Z 993304dc90b274b690e8ef820df2c963 diff --git a/manifest.uuid b/manifest.uuid index 611148833a..83f96a2ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2aa22509e7c8a1f09b16e4544c95d0b77503daed1f83106ccc18dee8bd9487a4 \ No newline at end of file +8822eb5d790bf8d4302c3f2dc05ab672193b309fb11771c4b3eea8a77f37e299 \ No newline at end of file diff --git a/test/check.test b/test/check.test index 2100aebb85..a6dbaa6171 100644 --- a/test/check.test +++ b/test/check.test @@ -484,6 +484,7 @@ do_catchsql_test 9.3 { # Integrity check on a VIEW with columns. # db close +db2 close forcedelete test.db sqlite3 db test.db do_execsql_test 10.1 { From 154cc746012d7fddd5436235e9f44ebd72a7ec6c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2017 15:11:40 +0000 Subject: [PATCH 268/292] Only run sync2.test on unix, as it depends on instrumentation in os_unix.c. FossilOrigin-Name: ecb9321e18dd72ea18d197c61c4d69500e9c4282c0eac67822cb40b2710a2815 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/sync2.test | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ad0fdedb4c..ed725cf65c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\scheck.test\sscript\sso\sthat\sit\sworks\son\sWindows. -D 2017-03-20T14:44:07.253 +C Only\srun\ssync2.test\son\sunix,\sas\sit\sdepends\son\sinstrumentation\sin\sos_unix.c. +D 2017-03-20T15:11:40.222 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1165,7 +1165,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 -F test/sync2.test 5ecf56be4c187a191253a1ec0786fe2df975dbf3dc0f0140cdf5d3690c90651c +F test/sync2.test c59ffd63e69854bf41ac407a9762ab5c128894b754253bf94db8a85d15de5444 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2aa22509e7c8a1f09b16e4544c95d0b77503daed1f83106ccc18dee8bd9487a4 -R 2a38f53db76a379174ef41cee348ccde -U drh -Z 993304dc90b274b690e8ef820df2c963 +P 8822eb5d790bf8d4302c3f2dc05ab672193b309fb11771c4b3eea8a77f37e299 +R 9d22d8b55292860d95cad32e3b689bec +U dan +Z 532b84081e4fb29bc1e9b05f7d75c0ea diff --git a/manifest.uuid b/manifest.uuid index 83f96a2ae7..18bf16e13c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8822eb5d790bf8d4302c3f2dc05ab672193b309fb11771c4b3eea8a77f37e299 \ No newline at end of file +ecb9321e18dd72ea18d197c61c4d69500e9c4282c0eac67822cb40b2710a2815 \ No newline at end of file diff --git a/test/sync2.test b/test/sync2.test index 5f9c9aba93..1fd74ef89b 100644 --- a/test/sync2.test +++ b/test/sync2.test @@ -26,6 +26,10 @@ ifcapable !pager_pragmas||!attach { finish_test return } +if {$::tcl_platform(platform)!="unix"} { + finish_test + return +} proc execsql_sync {sql} { set s $::sqlite_sync_count From 7adbcffcb94531498c2e43b246c4e3e4368691ef Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 15:29:28 +0000 Subject: [PATCH 269/292] Ensure that a "--" prefix is added to sqlite3_trace_v2() output for nested SQL statements. FossilOrigin-Name: 673a7b67c4828acaea3baebea500ef1f8ae763588b0d9c9f2ad6ed5ceb3cfee2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ed725cf65c..6e74d950bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\srun\ssync2.test\son\sunix,\sas\sit\sdepends\son\sinstrumentation\sin\sos_unix.c. -D 2017-03-20T15:11:40.222 +C Ensure\sthat\sa\s"--"\sprefix\sis\sadded\sto\ssqlite3_trace_v2()\soutput\sfor\snested\nSQL\sstatements. +D 2017-03-20T15:29:28.021 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -467,7 +467,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 3b4221473438a047d85839129c5e71eab80d8d19487c71e0ac6802fb330ae834 +F src/vdbe.c 89a12451405a17c6e8d39b5826acb6999f1283e4e43d2e83a7ac7c9a7094a86a F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8822eb5d790bf8d4302c3f2dc05ab672193b309fb11771c4b3eea8a77f37e299 -R 9d22d8b55292860d95cad32e3b689bec -U dan -Z 532b84081e4fb29bc1e9b05f7d75c0ea +P ecb9321e18dd72ea18d197c61c4d69500e9c4282c0eac67822cb40b2710a2815 +R 5262e66ba51f0f0c41b35ecdd4289055 +U drh +Z 6393fe64153a32f9de418b09796f1e18 diff --git a/manifest.uuid b/manifest.uuid index 18bf16e13c..ca8dc248ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecb9321e18dd72ea18d197c61c4d69500e9c4282c0eac67822cb40b2710a2815 \ No newline at end of file +673a7b67c4828acaea3baebea500ef1f8ae763588b0d9c9f2ad6ed5ceb3cfee2 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7eb2b6f6f7..93c20c0aaa 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6950,7 +6950,11 @@ case OP_Init: { /* jump */ sqlite3_free(z); }else #endif - { + if( db->nVdbeExec>1 ){ + char *z = sqlite3MPrintf(db, "-- %s", zTrace); + (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, z); + sqlite3DbFree(db, z); + }else{ (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); } } From 5b3a3b359a4f31b3776044e864f35557d5dcb2fb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2017 16:06:48 +0000 Subject: [PATCH 270/292] Do not run sync2.test as part of the "journaltest" permutation, as it uses "PRAGMA synchronous = off". FossilOrigin-Name: 285005a9bcb210bb2a9aa9fed6a19d4b78641a6e7622d469bd0d2a365b2c0735 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_journal.c | 3 +++ test/sync2.test | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6e74d950bd..1c7dda125d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sa\s"--"\sprefix\sis\sadded\sto\ssqlite3_trace_v2()\soutput\sfor\snested\nSQL\sstatements. -D 2017-03-20T15:29:28.021 +C Do\snot\srun\ssync2.test\sas\spart\sof\sthe\s"journaltest"\spermutation,\sas\sit\suses\n"PRAGMA\ssynchronous\s=\soff". +D 2017-03-20T16:06:48.281 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -435,7 +435,7 @@ F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64 F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c -F src/test_journal.c d3b83f2bcb7792c709e57abddc456a2b1818643a +F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c c05f6c40bd6c8bfe5f1718212f81fd5687f91766 F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e @@ -1165,7 +1165,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 -F test/sync2.test c59ffd63e69854bf41ac407a9762ab5c128894b754253bf94db8a85d15de5444 +F test/sync2.test b028aa9ca0bb4793e86140f777c6b88dcc6741f7a918381662e53bc5f2b97c74 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ecb9321e18dd72ea18d197c61c4d69500e9c4282c0eac67822cb40b2710a2815 -R 5262e66ba51f0f0c41b35ecdd4289055 -U drh -Z 6393fe64153a32f9de418b09796f1e18 +P 673a7b67c4828acaea3baebea500ef1f8ae763588b0d9c9f2ad6ed5ceb3cfee2 +R 547057b8e0709a92c08d29e340a54617 +U dan +Z c38b0bcff9e9a77e6d2ca931cd7486d0 diff --git a/manifest.uuid b/manifest.uuid index ca8dc248ab..7f9fef5671 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -673a7b67c4828acaea3baebea500ef1f8ae763588b0d9c9f2ad6ed5ceb3cfee2 \ No newline at end of file +285005a9bcb210bb2a9aa9fed6a19d4b78641a6e7622d469bd0d2a365b2c0735 \ No newline at end of file diff --git a/src/test_journal.c b/src/test_journal.c index 4e63bccf76..8a449e888c 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -557,6 +557,9 @@ static int jtWrite( u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==(int)p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); + /* The following assert() statements may fail if this layer is used + ** with a connection in "PRAGMA synchronous=off" mode. If they + ** fail with sync=normal or sync=full, this may indicate problem. */ assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); } diff --git a/test/sync2.test b/test/sync2.test index 1fd74ef89b..a88e4bed32 100644 --- a/test/sync2.test +++ b/test/sync2.test @@ -26,7 +26,7 @@ ifcapable !pager_pragmas||!attach { finish_test return } -if {$::tcl_platform(platform)!="unix"} { +if {$::tcl_platform(platform)!="unix" || [permutation] == "journaltest"} { finish_test return } From 5f42995a0a51be942cbb9095829767b175da0a87 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 16:34:18 +0000 Subject: [PATCH 271/292] Avoid the possibility of signed integer overflow with oversized precisions in %d conversions in the printf() implementation. FossilOrigin-Name: ef3a7c877a7549b351aafd983cfa96c863eb2641b6218bdd5cb563f659f879d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 5 +++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1c7dda125d..5c8fe10828 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\ssync2.test\sas\spart\sof\sthe\s"journaltest"\spermutation,\sas\sit\suses\n"PRAGMA\ssynchronous\s=\soff". -D 2017-03-20T16:06:48.281 +C Avoid\sthe\spossibility\sof\ssigned\sinteger\soverflow\swith\soversized\sprecisions\nin\s%d\sconversions\sin\sthe\sprintf()\simplementation. +D 2017-03-20T16:34:18.983 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -396,7 +396,7 @@ F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c 2b244434e76c7075edbcfd9e4d634899af0944ff01183b126d4671f7407c2368 F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a -F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c +F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 673a7b67c4828acaea3baebea500ef1f8ae763588b0d9c9f2ad6ed5ceb3cfee2 -R 547057b8e0709a92c08d29e340a54617 -U dan -Z c38b0bcff9e9a77e6d2ca931cd7486d0 +P 285005a9bcb210bb2a9aa9fed6a19d4b78641a6e7622d469bd0d2a365b2c0735 +R d28630b090e29d249202016ad6a6d827 +U drh +Z dd88d1a97157edc79245fb3d73eca0b6 diff --git a/manifest.uuid b/manifest.uuid index 7f9fef5671..68447cc550 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -285005a9bcb210bb2a9aa9fed6a19d4b78641a6e7622d469bd0d2a365b2c0735 \ No newline at end of file +ef3a7c877a7549b351aafd983cfa96c863eb2641b6218bdd5cb563f659f879d8 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 241338b266..a14e658875 100644 --- a/src/printf.c +++ b/src/printf.c @@ -400,12 +400,13 @@ void sqlite3VXPrintf( nOut = etBUFSIZE; zOut = buf; }else{ - nOut = precision + 10 + precision/3; - zOut = zExtra = sqlite3Malloc( nOut ); + u64 n = (u64)precision + 10 + precision/3; + zOut = zExtra = sqlite3Malloc( n ); if( zOut==0 ){ setStrAccumError(pAccum, STRACCUM_NOMEM); return; } + nOut = (int)n; } bufpt = &zOut[nOut-1]; if( xtype==etORDINAL ){ From 920c83f18fb8cd749774a63ce1a062d07ff56220 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2017 18:53:32 +0000 Subject: [PATCH 272/292] Fix some problems in fts3 found by address-sanitizer. FossilOrigin-Name: 16a8e84fa7f67a467f824bdd7f72cbd6a6e95dab8cc7aa1e0e751720b98f3e31 --- ext/fts3/fts3.c | 5 +++-- ext/fts3/fts3_unicode.c | 26 +++++++++++++------------- ext/fts3/fts3_unicode2.c | 21 ++++++++++----------- ext/fts3/fts3_write.c | 5 ++++- ext/fts3/unicode/mkunicode.tcl | 2 +- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- 7 files changed, 43 insertions(+), 40 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 8af62cda45..8abc338045 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; u32 a; u64 b; int shift; diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 94fc27b5b4..dfb2680c50 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -136,16 +136,16 @@ static int unicodeAddExceptions( ){ const unsigned char *z = (const unsigned char *)zIn; const unsigned char *zTerm = &z[nIn]; - int iCode; + unsigned int iCode; int nEntry = 0; assert( bAlnum==0 || bAlnum==1 ); while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = iCode; + aNew[i] = (int)iCode; nNew++; } } @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode = 0; + unsigned int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -330,7 +330,7 @@ static int unicodeNext( ** the input. */ while( z=zTerm ) return SQLITE_DONE; @@ -350,7 +350,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } @@ -358,8 +358,8 @@ static int unicodeNext( /* If the cursor is not at EOF, read the next character */ if( z>=zTerm ) break; READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, iCode) - || sqlite3FtsUnicodeIsdiacritic(iCode) + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) ); /* Set the output variables and return. */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index 20b7a25dbf..da7251ed0c 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, }; - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; @@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ int ret = c; - assert( c>=0 ); assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); if( c<128 ){ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); }else if( c<65536 ){ + const struct TableEntry *p; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; int iRes = -1; + assert( c>aEntry[0].iCode ); while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; int cmp = (c - aEntry[iTest].iCode); @@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ iHi = iTest-1; } } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } if( bRemoveDiacritic ) ret = remove_diacritic(ret); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3ff481b0b0..d700c8403c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4956,11 +4956,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** Convert the text beginning at *pz into an integer and return ** its value. Advance *pz to point to the first character past ** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. */ static int fts3Getint(const char **pz){ const char *z = *pz; int i = 0; - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; *pz = z; return i; } diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index aafb4e9f9b..de89099122 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -227,7 +227,7 @@ proc print_isalnum {zFunc lRange} { an_print_ascii_bitmap $lRange puts { if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; diff --git a/manifest b/manifest index 5c8fe10828..882e3cc273 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sthe\spossibility\sof\ssigned\sinteger\soverflow\swith\soversized\sprecisions\nin\s%d\sconversions\sin\sthe\sprintf()\simplementation. -D 2017-03-20T16:34:18.983 +C Fix\ssome\sproblems\sin\sfts3\sfound\sby\saddress-sanitizer. +D 2017-03-20T18:53:32.346 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 95c7041ea75d82d2d9a4cd058904ba889751f5b8 +F ext/fts3/fts3.c df964dbcc7bb97b8871c53866eb6c4e9c05c0d8af42038fa6439fb411260dd2e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -86,15 +86,15 @@ F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 -F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 -F ext/fts3/fts3_write.c c3863f23b6b4623c8b9d5cf31c12ce4469f78ca9 +F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d +F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c +F ext/fts3/fts3_write.c a51d48d646974ee2fb4b17fcd5da0416a5759a32dcacc2cce2ba00d5a767848e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 +F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c4951a0fbb60e780cb69f F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 285005a9bcb210bb2a9aa9fed6a19d4b78641a6e7622d469bd0d2a365b2c0735 -R d28630b090e29d249202016ad6a6d827 -U drh -Z dd88d1a97157edc79245fb3d73eca0b6 +P ef3a7c877a7549b351aafd983cfa96c863eb2641b6218bdd5cb563f659f879d8 +R 646761e50bb7679c40f23d2ddea8c902 +U dan +Z f4376c790f10e78888f80aca2d0f09f6 diff --git a/manifest.uuid b/manifest.uuid index 68447cc550..c4f06f22c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef3a7c877a7549b351aafd983cfa96c863eb2641b6218bdd5cb563f659f879d8 \ No newline at end of file +16a8e84fa7f67a467f824bdd7f72cbd6a6e95dab8cc7aa1e0e751720b98f3e31 \ No newline at end of file From 6b904f5e01bb2737b093b154b6ab39b393daa0f6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2017 19:26:27 +0000 Subject: [PATCH 273/292] Avoid a technically undefined right-shift of a signed value in rtree.c. FossilOrigin-Name: a144875fe44ff3a30bab299d50b7dbec2ee21f8c73e692a71ee1f7c54b5f0c76 --- ext/rtree/rtree.c | 18 +++++++++--------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 63838a4eef..42f08a9628 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -458,15 +458,15 @@ static i64 readInt64(u8 *p){ memcpy(&x, p, 8); return x; #else - return ( - (((i64)p[0]) << 56) + - (((i64)p[1]) << 48) + - (((i64)p[2]) << 40) + - (((i64)p[3]) << 32) + - (((i64)p[4]) << 24) + - (((i64)p[5]) << 16) + - (((i64)p[6]) << 8) + - (((i64)p[7]) << 0) + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) ); #endif } diff --git a/manifest b/manifest index 882e3cc273..180628583d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\sin\sfts3\sfound\sby\saddress-sanitizer. -D 2017-03-20T18:53:32.346 +C Avoid\sa\stechnically\sundefined\sright-shift\sof\sa\ssigned\svalue\sin\srtree.c. +D 2017-03-20T19:26:27.669 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -270,7 +270,7 @@ F ext/rbu/sqlite3rbu.c 2a89efba9eeba8e6c89a498dc195e8efbdde2694 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 3f3a595dba485e340246fa2c8ba330a6b9768b00 +F ext/rtree/rtree.c 0acd285bfacc347579a5df9fe947212fb99e2775a40c43f027c3a16936c58e7e F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef3a7c877a7549b351aafd983cfa96c863eb2641b6218bdd5cb563f659f879d8 -R 646761e50bb7679c40f23d2ddea8c902 +P 16a8e84fa7f67a467f824bdd7f72cbd6a6e95dab8cc7aa1e0e751720b98f3e31 +R 3866887b2e2a6895d2c139049fd577ac U dan -Z f4376c790f10e78888f80aca2d0f09f6 +Z 940954dc69a4e0a1139a0110a41a8583 diff --git a/manifest.uuid b/manifest.uuid index c4f06f22c9..97c8d21348 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16a8e84fa7f67a467f824bdd7f72cbd6a6e95dab8cc7aa1e0e751720b98f3e31 \ No newline at end of file +a144875fe44ff3a30bab299d50b7dbec2ee21f8c73e692a71ee1f7c54b5f0c76 \ No newline at end of file From 74869e5e52c56bab031e190061afd5d531d88f36 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2017 19:35:30 +0000 Subject: [PATCH 274/292] Avoid passing NULL as the second argument to memcpy() in fts3. FossilOrigin-Name: 49b93d972de9649abfd6235b65dda1c9d468956671e50704afde6181ffa56dea --- ext/fts3/fts3.c | 4 +++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 8abc338045..6f059bc95d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1398,7 +1398,9 @@ static int fts3InitVtab( char *z; int n = 0; z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n); - memcpy(zCsr, z, n); + if( n>0 ){ + memcpy(zCsr, z, n); + } zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; diff --git a/manifest b/manifest index 180628583d..9fbd812c57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\stechnically\sundefined\sright-shift\sof\sa\ssigned\svalue\sin\srtree.c. -D 2017-03-20T19:26:27.669 +C Avoid\spassing\sNULL\sas\sthe\ssecond\sargument\sto\smemcpy()\sin\sfts3. +D 2017-03-20T19:35:30.725 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c df964dbcc7bb97b8871c53866eb6c4e9c05c0d8af42038fa6439fb411260dd2e +F ext/fts3/fts3.c f0d5de1bc2155ba7cd7c0c1a751779a3a8857fa34d5c12f3b233a23fa2e79ea2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 16a8e84fa7f67a467f824bdd7f72cbd6a6e95dab8cc7aa1e0e751720b98f3e31 -R 3866887b2e2a6895d2c139049fd577ac +P a144875fe44ff3a30bab299d50b7dbec2ee21f8c73e692a71ee1f7c54b5f0c76 +R bd3c42588251ebb6d9905386e3af9ca1 U dan -Z 940954dc69a4e0a1139a0110a41a8583 +Z a06ad4d721634b829ae52c87102c1626 diff --git a/manifest.uuid b/manifest.uuid index 97c8d21348..cd1cb6be7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a144875fe44ff3a30bab299d50b7dbec2ee21f8c73e692a71ee1f7c54b5f0c76 \ No newline at end of file +49b93d972de9649abfd6235b65dda1c9d468956671e50704afde6181ffa56dea \ No newline at end of file From 16f0582c1b0f059e27df99bf3648d525a3d76dc7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 20:42:21 +0000 Subject: [PATCH 275/292] Fix the fuzzcheck program so that it can create new databases again. FossilOrigin-Name: 021e8874a7d1bb94debae3ae04f83056a8573148ffc872cd76a186a2d22d0296 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzzcheck.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9fbd812c57..6ce523a998 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\spassing\sNULL\sas\sthe\ssecond\sargument\sto\smemcpy()\sin\sfts3. -D 2017-03-20T19:35:30.725 +C Fix\sthe\sfuzzcheck\sprogram\sso\sthat\sit\scan\screate\snew\sdatabases\sagain. +D 2017-03-20T20:42:21.876 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -833,7 +833,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c a87e6067a8d19844bade916841cb76150ecf24a2 +F test/fuzzcheck.c 0cfce51bd5b0b7ab5d2d2ecaf9a82d5962f90852348be6bf5f05a1fb2aaf40e0 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a144875fe44ff3a30bab299d50b7dbec2ee21f8c73e692a71ee1f7c54b5f0c76 -R bd3c42588251ebb6d9905386e3af9ca1 -U dan -Z a06ad4d721634b829ae52c87102c1626 +P 49b93d972de9649abfd6235b65dda1c9d468956671e50704afde6181ffa56dea +R 26d368e43476d2b80e7af8c6d59964fd +U drh +Z 95628c590b2b14daadfe6003e34f6d94 diff --git a/manifest.uuid b/manifest.uuid index cd1cb6be7f..db25339f3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49b93d972de9649abfd6235b65dda1c9d468956671e50704afde6181ffa56dea \ No newline at end of file +021e8874a7d1bb94debae3ae04f83056a8573148ffc872cd76a186a2d22d0296 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 75d519566f..2efe68440a 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -972,7 +972,7 @@ int main(int argc, char **argv){ /* Process each source database separately */ for(iSrcDb=0; iSrcDbzName); + SQLITE_OPEN_READWRITE, pDfltVfs->zName); if( rc ){ fatalError("cannot open source database %s - %s", azSrcDb[iSrcDb], sqlite3_errmsg(db)); @@ -1047,6 +1047,8 @@ int main(int argc, char **argv){ sqlite3_close(db); return 0; } + rc = sqlite3_exec(db, "PRAGMA query_only=1;", 0, 0, 0); + if( rc ) fatalError("cannot set database to query-only"); if( zExpDb!=0 || zExpSql!=0 ){ sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); From 174f8553068e086e2b7b62bee89d3a8eee6f087a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Mar 2017 22:58:27 +0000 Subject: [PATCH 276/292] Add the --native-malloc option to fuzzcheck. Fix ossfuzz.c and fuzzcheck.c so that they both deallocate the temp_store_directory before closing. FossilOrigin-Name: 0dd18ec882bb28a87629d6d8dfeb5ea5d82833634b3781a7d14b917272c4dfa0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzzcheck.c | 22 ++++++++++++++++++---- test/ossfuzz.c | 1 + 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6ce523a998..e16247c45b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sfuzzcheck\sprogram\sso\sthat\sit\scan\screate\snew\sdatabases\sagain. -D 2017-03-20T20:42:21.876 +C Add\sthe\s--native-malloc\soption\sto\sfuzzcheck.\s\sFix\sossfuzz.c\sand\sfuzzcheck.c\nso\sthat\sthey\sboth\sdeallocate\sthe\stemp_store_directory\sbefore\sclosing. +D 2017-03-20T22:58:27.453 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -833,7 +833,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c 0cfce51bd5b0b7ab5d2d2ecaf9a82d5962f90852348be6bf5f05a1fb2aaf40e0 +F test/fuzzcheck.c 2152602232c96d9c790eff3013e1369ce59de3203fa0b75bc613531448454e61 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1009,7 +1009,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f -F test/ossfuzz.c 8c4d62e156352ecb97eb868fcff20d828a279af67a1c1310779f63464d6f009f +F test/ossfuzz.c f5abed3177f719df3c3109901fcdd26b9fb7f581c8da50fc26f3a81ddfb2c2ae F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49b93d972de9649abfd6235b65dda1c9d468956671e50704afde6181ffa56dea -R 26d368e43476d2b80e7af8c6d59964fd +P 021e8874a7d1bb94debae3ae04f83056a8573148ffc872cd76a186a2d22d0296 +R 694ec38d9cd79a3b1927a92dd170b999 U drh -Z 95628c590b2b14daadfe6003e34f6d94 +Z 6cb1e54838f3a51f8b4c2a137da28ce9 diff --git a/manifest.uuid b/manifest.uuid index db25339f3b..ad7fcf00b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -021e8874a7d1bb94debae3ae04f83056a8573148ffc872cd76a186a2d22d0296 \ No newline at end of file +0dd18ec882bb28a87629d6d8dfeb5ea5d82833634b3781a7d14b917272c4dfa0 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 2efe68440a..fd72273f34 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -805,6 +805,7 @@ static void showHelp(void){ " --load-db ARGS... Load template databases from files into SOURCE_DB\n" " -m TEXT Add a description to the database\n" " --native-vfs Use the native VFS for initially empty database files\n" +" --native-malloc Turn off MEMSYS3/5 and Lookaside\n" " --oss-fuzz Enable OSS-FUZZ testing\n" " --prng-seed N Seed value for the PRGN inside of SQLite\n" " --rebuild Rebuild and vacuum the database file\n" @@ -851,6 +852,7 @@ int main(int argc, char **argv){ void *pHeap = 0; /* Heap for use by SQLite */ int ossFuzz = 0; /* enable OSS-FUZZ testing */ int ossFuzzThisDb = 0; /* ossFuzz value for this particular database */ + int nativeMalloc = 0; /* Turn off MEMSYS3/5 and lookaside if true */ sqlite3_vfs *pDfltVfs; /* The default VFS */ iBegin = timeOfDay(); @@ -911,6 +913,9 @@ int main(int argc, char **argv){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); zMsg = argv[++i]; }else + if( strcmp(z,"native-malloc")==0 ){ + nativeMalloc = 1; + }else if( strcmp(z,"native-vfs")==0 ){ nativeFlag = 1; }else @@ -1013,7 +1018,7 @@ int main(int argc, char **argv){ ossFuzzThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); } - if( strcmp(zName, "limit-mem")==0 ){ + if( strcmp(zName, "limit-mem")==0 && !nativeMalloc ){ #if !defined(SQLITE_ENABLE_MEMSYS3) && !defined(SQLITE_ENABLE_MEMSYS5) fatalError("the limit-mem option requires -DSQLITE_ENABLE_MEMSYS5" " or _MEMSYS3"); @@ -1143,14 +1148,19 @@ int main(int argc, char **argv){ } /* Limit available memory, if requested */ - if( nMemThisDb>0 ){ - sqlite3_shutdown(); + sqlite3_shutdown(); + if( nMemThisDb>0 && !nativeMalloc ){ pHeap = realloc(pHeap, nMemThisDb); if( pHeap==0 ){ fatalError("failed to allocate %d bytes of heap memory", nMem); } sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); } + + /* Disable lookaside with the --native-malloc option */ + if( nativeMalloc ){ + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); + } /* Reset the in-memory virtual filesystem */ formatVfs(); @@ -1207,9 +1217,13 @@ int main(int argc, char **argv){ runSql(db, (char*)pSql->a, runFlags); }while( timeoutTest ); setAlarm(0); + sqlite3_exec(db, "PRAGMA temp_store_directory=''", 0, 0, 0); sqlite3_close(db); } - if( sqlite3_memory_used()>0 ) fatalError("memory leak"); + if( sqlite3_memory_used()>0 ){ + fatalError("memory leak: %lld bytes outstanding", + sqlite3_memory_used()); + } reformatVfs(); nTest++; g.zTestName[0] = 0; diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 0c2b1320e4..51983548b9 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -145,6 +145,7 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* Cleanup and return */ sqlite3_free(zErrMsg); sqlite3_free(zSql); + sqlite3_exec(cx.db, "PRAGMA temp_store_directory=''", 0, 0, 0); sqlite3_close(cx.db); if( mDebug & FUZZ_SHOW_MAX_DELAY ){ From a14b7e02e97aecc5712d826e1e7936e6a9219dd2 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Mar 2017 10:45:38 +0000 Subject: [PATCH 277/292] Do not run sync2.test as part of the "inmemory_journal" permutation. FossilOrigin-Name: 9f680bc7c47f8391adba6ea2b4b2cb6a5a041e22753543d5224da6656c3762fe --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/sync2.test | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e16247c45b..f6d020fb32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--native-malloc\soption\sto\sfuzzcheck.\s\sFix\sossfuzz.c\sand\sfuzzcheck.c\nso\sthat\sthey\sboth\sdeallocate\sthe\stemp_store_directory\sbefore\sclosing. -D 2017-03-20T22:58:27.453 +C Do\snot\srun\ssync2.test\sas\spart\sof\sthe\s"inmemory_journal"\spermutation. +D 2017-03-21T10:45:38.479 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -1165,7 +1165,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 -F test/sync2.test b028aa9ca0bb4793e86140f777c6b88dcc6741f7a918381662e53bc5f2b97c74 +F test/sync2.test 29af00fe9e468a1e20315d1edced8f8a32ac156df983ae9b53802e7d472675d5 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 @@ -1566,7 +1566,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 021e8874a7d1bb94debae3ae04f83056a8573148ffc872cd76a186a2d22d0296 -R 694ec38d9cd79a3b1927a92dd170b999 -U drh -Z 6cb1e54838f3a51f8b4c2a137da28ce9 +P 0dd18ec882bb28a87629d6d8dfeb5ea5d82833634b3781a7d14b917272c4dfa0 +R 8db91ad0b42e6ccb8a27a4c262ebc723 +U dan +Z 688065de24febc90cf37734d50aa7e01 diff --git a/manifest.uuid b/manifest.uuid index ad7fcf00b1..f7dfbdc10c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dd18ec882bb28a87629d6d8dfeb5ea5d82833634b3781a7d14b917272c4dfa0 \ No newline at end of file +9f680bc7c47f8391adba6ea2b4b2cb6a5a041e22753543d5224da6656c3762fe \ No newline at end of file diff --git a/test/sync2.test b/test/sync2.test index a88e4bed32..4f6ed7754c 100644 --- a/test/sync2.test +++ b/test/sync2.test @@ -26,7 +26,10 @@ ifcapable !pager_pragmas||!attach { finish_test return } -if {$::tcl_platform(platform)!="unix" || [permutation] == "journaltest"} { +if {$::tcl_platform(platform)!="unix" + || [permutation] == "journaltest" + || [permutation] == "inmemory_journal" +} { finish_test return } From 119fc11eb0b746495df92698fce9696591293519 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Mar 2017 17:19:31 +0000 Subject: [PATCH 278/292] Add short script ext/fts3/tool/fts3cov.sh. To help measure test-coverage of fts3 source code. FossilOrigin-Name: ee9588e873ffebcaa177957950cbb14924e154c391ed7f687116065064ff11b0 --- ext/fts3/tool/fts3cov.sh | 16 ++++++++++++++++ manifest | 13 +++++++------ manifest.uuid | 2 +- test/permutations.test | 34 ++++++++++++++++++---------------- 4 files changed, 42 insertions(+), 23 deletions(-) create mode 100644 ext/fts3/tool/fts3cov.sh diff --git a/ext/fts3/tool/fts3cov.sh b/ext/fts3/tool/fts3cov.sh new file mode 100644 index 0000000000..b1f34dce76 --- /dev/null +++ b/ext/fts3/tool/fts3cov.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +srcdir=`dirname $(dirname $(dirname $(dirname $0)))` +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt + +echo "" + +for f in `ls $srcdir/ext/fts3/*.c` +do + f=`basename $f` + echo -ne "$f: " + gcov -b $f | grep Taken | sed 's/Taken at least once://' +done + diff --git a/manifest b/manifest index f6d020fb32..ec8ad84599 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\ssync2.test\sas\spart\sof\sthe\s"inmemory_journal"\spermutation. -D 2017-03-21T10:45:38.479 +C Add\sshort\sscript\sext/fts3/tool/fts3cov.sh.\sTo\shelp\smeasure\stest-coverage\sof\nfts3\ssource\scode. +D 2017-03-21T17:19:31.218 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -91,6 +91,7 @@ F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75 F ext/fts3/fts3_write.c a51d48d646974ee2fb4b17fcd5da0416a5759a32dcacc2cce2ba00d5a767848e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 +F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 @@ -1025,7 +1026,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 46ad98770d3c04ce2418cab2cc527647bfe961a7 +F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1566,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0dd18ec882bb28a87629d6d8dfeb5ea5d82833634b3781a7d14b917272c4dfa0 -R 8db91ad0b42e6ccb8a27a4c262ebc723 +P 9f680bc7c47f8391adba6ea2b4b2cb6a5a041e22753543d5224da6656c3762fe +R 6048d4479132e9b80b95aca588407d4c U dan -Z 688065de24febc90cf37734d50aa7e01 +Z d545083811aa9af855b84bbadd9a61dc diff --git a/manifest.uuid b/manifest.uuid index f7dfbdc10c..c60120ecbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f680bc7c47f8391adba6ea2b4b2cb6a5a041e22753543d5224da6656c3762fe \ No newline at end of file +ee9588e873ffebcaa177957950cbb14924e154c391ed7f687116065064ff11b0 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 7f81dcbb27..628e7ba44e 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -247,22 +247,24 @@ test_suite "threads" -prefix "" -description { test_suite "fts3" -prefix "" -description { All FTS3 tests except fts3rnd.test. } -files { - fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test - fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test - fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test - fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test - fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test - fts3expr3.test - fts3near.test fts3query.test fts3shared.test fts3snippet.test - fts3sort.test - fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test fts3auto.test - fts4aa.test fts4content.test - fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test - fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test - fts4check.test fts4unicode.test fts4noti.test - fts3varint.test - fts4growth.test fts4growth2.test + fts3aa.test fts3ab.test fts3ac.test fts3ad.test + fts3ae.test fts3af.test fts3ag.test fts3ah.test + fts3ai.test fts3aj.test fts3ak.test fts3al.test + fts3am.test fts3an.test fts3ao.test fts3atoken.test + fts3auto.test fts3aux1.test fts3aux2.test fts3b.test + fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test + fts3cov.test fts3c.test fts3defer2.test fts3defer3.test + fts3defer.test fts3drop.test fts3d.test fts3e.test + fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test + fts3expr.test fts3fault2.test fts3fault.test fts3first.test + fts3join.test fts3malloc.test fts3matchinfo.test fts3near.test + fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test + fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test + fts3tok_err.test fts3varint.test fts4aa.test fts4check.test + fts4content.test fts4docid.test fts4growth2.test fts4growth.test + fts4incr.test fts4langid.test fts4lastrowid.test fts4merge2.test + fts4merge4.test fts4merge.test fts4noti.test fts4onepass.test + fts4opt.test fts4unicode.test } test_suite "fts5" -prefix "" -description { From 49297c86d2f9db1944fd84d00f5cfbfa56fdb1bf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Mar 2017 18:56:52 +0000 Subject: [PATCH 279/292] Fix an incorrect assert in the ANALYZE logic for STAT4 on WITHOUT ROWID tables. FossilOrigin-Name: ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ec8ad84599..75a4357e21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sshort\sscript\sext/fts3/tool/fts3cov.sh.\sTo\shelp\smeasure\stest-coverage\sof\nfts3\ssource\scode. -D 2017-03-21T17:19:31.218 +C Fix\san\sincorrect\sassert\sin\sthe\sANALYZE\slogic\sfor\sSTAT4\son\sWITHOUT\sROWID\ntables. +D 2017-03-21T18:56:52.218 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -338,7 +338,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 6d8234916c29be943e6ea28b5bef67dff98d9905 +F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f680bc7c47f8391adba6ea2b4b2cb6a5a041e22753543d5224da6656c3762fe -R 6048d4479132e9b80b95aca588407d4c -U dan -Z d545083811aa9af855b84bbadd9a61dc +P ee9588e873ffebcaa177957950cbb14924e154c391ed7f687116065064ff11b0 +R 5a75fd33dd28f00ec2c9a64301b362e9 +U drh +Z a2519e8e7a444d8c646c2457bd4b758f diff --git a/manifest.uuid b/manifest.uuid index c60120ecbc..e7b01a944f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee9588e873ffebcaa177957950cbb14924e154c391ed7f687116065064ff11b0 \ No newline at end of file +ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d1df000943..495cc954ac 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1204,7 +1204,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knCol ); + assert( k>=0 && knColumn ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } From 918938f9c205b7fc99b6abd66bfaad1d1895d694 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Mar 2017 20:17:24 +0000 Subject: [PATCH 280/292] New simplified memory initialization for MacOS. FossilOrigin-Name: 055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem1.c | 48 +++++++++++++----------------------------------- 3 files changed, 20 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 75a4357e21..834fc1ba0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassert\sin\sthe\sANALYZE\slogic\sfor\sSTAT4\son\sWITHOUT\sROWID\ntables. -D 2017-03-21T18:56:52.218 +C New\ssimplified\smemory\sinitialization\sfor\sMacOS. +D 2017-03-21T20:17:24.121 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -369,7 +369,7 @@ F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 79bf195f445bf7e66cadd121849837c3152fbd2f542326bbed3073b6902450c2 +F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee9588e873ffebcaa177957950cbb14924e154c391ed7f687116065064ff11b0 -R 5a75fd33dd28f00ec2c9a64301b362e9 +P ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d +R c272fa429602630b64ee19437bd8d503 U drh -Z a2519e8e7a444d8c646c2457bd4b758f +Z a3a73b80a262fb4daeb6a94d5a29c210 diff --git a/manifest.uuid b/manifest.uuid index e7b01a944f..e5a19cd1b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d \ No newline at end of file +055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index 02ed59b4d4..512ab3747f 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -238,45 +238,23 @@ static int sqlite3MemRoundup(int n){ */ static int sqlite3MemInit(void *NotUsed){ #if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + int cpuCount; + size_t len; if( _sqliteZone_ ){ return SQLITE_OK; } -#ifndef SQLITE_MIGHT_BE_SINGLE_CORE - /* If not compiled with the SQLITE_MIGHT_BE_SINGLE_CORE flag, assume - ** that multiple cores are always available. This is the default case. - */ - _sqliteZone_ = malloc_default_zone(); -#else - /* With the SQLITE_MIGHT_BE_SINGLE_CORE compile-time option, check the - ** number of cores. Different malloc() strategies are used for single-core and - ** multi-core machines. - */ - { - int cpuCount; - size_t len; - len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ - sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); - if( cpuCount>1 ){ - /* defer MT decisions to system malloc */ - _sqliteZone_ = malloc_default_zone(); - }else{ - /* only 1 core, use our own zone to contention over global locks, - ** e.g. we have our own dedicated locks */ - bool success; - malloc_zone_t* newzone = malloc_create_zone(4096, 0); - malloc_set_zone_name(newzone, "Sqlite_Heap"); - do{ - success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, - (void * volatile *)&_sqliteZone_); - }while(!_sqliteZone_); - if( !success ){ - /* somebody registered a zone first */ - malloc_destroy_zone(newzone); - } - } + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); } -#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ #endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ UNUSED_PARAMETER(NotUsed); return SQLITE_OK; From 701ff6ad2d00fe7f05a4de4e946a2a4893b1c722 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Mar 2017 12:51:34 +0000 Subject: [PATCH 281/292] Fix harmless compiler warnings in the shell. FossilOrigin-Name: a786829783ef65ef270ca360712364cbd47a540d31ed1547d50808aad698bea7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 834fc1ba0f..560760071c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\ssimplified\smemory\sinitialization\sfor\sMacOS. -D 2017-03-21T20:17:24.121 +C Fix\sharmless\scompiler\swarnings\sin\sthe\sshell. +D 2017-03-22T12:51:34.759 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -402,7 +402,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa -F src/shell.c 77054c021069ec0b65d3d620aab031f97c59b4e42ac7c31544ea68933b581104 +F src/shell.c ce39c4bbbaa02484cb45141034fd363c3bac80a6aa22b0bb3cecea1c6a4c0979 F src/sqlite.h.in 723107d97f2345a7c103632169dc61366121c4ab65d75a7d83c6dc0e5bbe5ca4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d -R c272fa429602630b64ee19437bd8d503 +P 055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 +R 1f280e53b7c950d24565e873ae0276ec U drh -Z a3a73b80a262fb4daeb6a94d5a29c210 +Z a154993c25ee99ce8fe87b6f82cb272d diff --git a/manifest.uuid b/manifest.uuid index e5a19cd1b9..6c408aa251 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 \ No newline at end of file +a786829783ef65ef270ca360712364cbd47a540d31ed1547d50808aad698bea7 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e7652f8e67..e71c34e6df 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2926,7 +2926,6 @@ static char **tableColumnList(ShellState *p, const char *zTab){ ** ordinary column in the table. Verify that azRowid[j] is a valid ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID ** tables will fail this last check */ - int rc; rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; break; @@ -2959,14 +2958,14 @@ static void toggleSelectOrder(sqlite3 *db){ ** the table type ("index" or "table") and SQL to create the table. ** This routine should print text sufficient to recreate the table. */ -static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ int rc; const char *zTable; const char *zType; const char *zSql; ShellState *p = (ShellState *)pArg; - UNUSED_PARAMETER(azCol); + UNUSED_PARAMETER(azNotUsed); if( nArg!=3 ) return 1; zTable = azArg[0]; zType = azArg[1]; From 633647af75b3e680abe57922a8f1ec495b4150c6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Mar 2017 21:24:31 +0000 Subject: [PATCH 282/292] Initial implementation of the json_mergepatch(A,B) SQL function. FossilOrigin-Name: a267444039af519f088dd8f8ee33f686cc3071c087677075af2364ebc2587514 --- ext/misc/json1.c | 138 +++++++++++++++++++++++++++++++++++++++------- manifest | 16 ++++-- manifest.uuid | 2 +- test/json104.test | 63 +++++++++++++++++++++ 4 files changed, 192 insertions(+), 27 deletions(-) create mode 100644 test/json104.test diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 3063f9f261..0db946fdc6 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -138,9 +138,10 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ /* A single node of parsed JSON @@ -148,12 +149,13 @@ static const char * const jsonType[] = { struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ - u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ union { const char *zJContent; /* Content for INT, REAL, and STRING */ u32 iAppend; /* More terms for ARRAY and OBJECT */ u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ } u; }; @@ -410,6 +412,13 @@ static void jsonRenderNode( JsonString *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } switch( pNode->eType ){ default: { assert( pNode->eType==JSON_NULL ); @@ -441,12 +450,7 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ - jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); - } - }else{ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } @@ -468,11 +472,7 @@ static void jsonRenderNode( jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - }else{ - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } + jsonRenderNode(&pNode[j+1], pOut, aReplace); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -699,7 +699,6 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; - p->iVal = 0; p->n = n; p->u.zJContent = zContent; return pParse->nNode++; @@ -1357,6 +1356,104 @@ static void jsonExtractFunc( jsonParseReset(&x); } +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + int iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + int i, j; + int iApnd; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + for(i=2; in; i += jsonNodeSize(&pPatch[i])+1){ + if( pPatch[i].eType==JSON_NULL ){ + pPatch[i-1].jnFlags |= JNODE_REMOVE; + } + } + return pPatch; + } + iApnd = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + int nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + if( (pPatch[i].jnFlags & JNODE_RAW)==0 ){ + assert( nKey>=2 && zKey[0]=='"' && zKey[nKey-1]=='"' ); + nKey -= 2; + zKey ++; + } + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + if( jsonLabelCompare(&pTarget[j], zKey, nKey) + && (pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH))==0 + ){ + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iApnd].jnFlags |= JNODE_APPEND; + pParse->aNode[iApnd].u.iAppend = iStart; + iApnd = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonMergePatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + jsonReturnJson(jsonMergePatch(&x, 0, y.aNode), ctx, 0); + jsonParseReset(&x); + jsonParseReset(&y); +} + + /* ** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON ** object that contains all name/value given in arguments. Or if any name @@ -1460,11 +1557,11 @@ static void jsonReplaceFunc( if( x.nErr ) goto replace_err; if( pNode ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -1514,11 +1611,11 @@ static void jsonSetFunc( goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -2160,6 +2257,7 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_array_length", 2, 0, jsonArrayLengthFunc }, { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, + { "json_mergepatch", 2, 0, jsonMergePatchFunc }, { "json_object", -1, 0, jsonObjectFunc }, { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, diff --git a/manifest b/manifest index 834fc1ba0f..cceedf00d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\ssimplified\smemory\sinitialization\sfor\sMacOS. -D 2017-03-21T20:17:24.121 +C Initial\simplementation\sof\sthe\sjson_mergepatch(A,B)\sSQL\sfunction. +D 2017-03-22T21:24:31.714 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c 552a7d730863419e05dc947265b28bd411680e2e +F ext/misc/json1.c ca27a98c0a7a90fcbaccd3157204e19afc7eec0ba3f4c08ed06bb638b773f523 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -913,6 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 +F test/json104.test 83fd7a15eadb0cde34a37200842318d1cd98abe908e2847fb93d337d969815cc F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1567,7 +1568,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad741976c8c29bcc94f9ea9ed7deb580bb00c8a81d1a7fba1a4e03849728433d -R c272fa429602630b64ee19437bd8d503 +P 055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 +R 1d5a57aea0e07c508840a776caf26b14 +T *branch * json_mergepatch +T *sym-json_mergepatch * +T -sym-trunk * U drh -Z a3a73b80a262fb4daeb6a94d5a29c210 +Z 2961dcef6bde9e861663934a25c90d24 diff --git a/manifest.uuid b/manifest.uuid index e5a19cd1b9..980cac0acf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 \ No newline at end of file +a267444039af519f088dd8f8ee33f686cc3071c087677075af2364ebc2587514 \ No newline at end of file diff --git a/test/json104.test b/test/json104.test new file mode 100644 index 0000000000..fc2e0e38b4 --- /dev/null +++ b/test/json104.test @@ -0,0 +1,63 @@ +# 2017-03-22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for json_mergepatch(A,B) SQL function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !json1 { + finish_test + return +} + +# This is the example from pages 2 and 3 of RFC-7396 +do_execsql_test json104-100 { + SELECT json_mergepatch( + json('{ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + }'), + json('{ + "a":"z", + "c": { + "f": null + } + }')); +} {{{"a":"z","c":{"d":"e"}}}} + + +# This is the example from pages 4 and 5 of RFC-7396 +do_execsql_test json104-110 { + SELECT json_mergepatch( + json('{ + "title": "Goodbye!", + "author" : { + "givenName" : "John", + "familyName" : "Doe" + }, + "tags":[ "example", "sample" ], + "content": "This will be unchanged" + }'), + json('{ + "title": "Hello!", + "phoneNumber": "+01-123-456-7890", + "author": { + "familyName": null + }, + "tags": [ "example" ] + }')); +} {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged",phoneNumber:"+01-123-456-7890"}}} + +finish_test From f07b249f92fb1d0a37b1444657d14ed7042bafa7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Mar 2017 21:45:20 +0000 Subject: [PATCH 283/292] Change the name of the new function to "json_merge_patch()". FossilOrigin-Name: 53bf70f37bbca319ea35f70849e2a34ae628a504486158fdad5c4bb7431c68e0 --- ext/misc/json1.c | 2 +- manifest | 17 +++++++---------- manifest.uuid | 2 +- test/json104.test | 16 ++++++---------- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 0db946fdc6..c037525c45 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -2257,7 +2257,7 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_array_length", 2, 0, jsonArrayLengthFunc }, { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, - { "json_mergepatch", 2, 0, jsonMergePatchFunc }, + { "json_merge_patch", 2, 0, jsonMergePatchFunc }, { "json_object", -1, 0, jsonObjectFunc }, { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, diff --git a/manifest b/manifest index cceedf00d1..8c6f009d42 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\sthe\sjson_mergepatch(A,B)\sSQL\sfunction. -D 2017-03-22T21:24:31.714 +C Change\sthe\sname\sof\sthe\snew\sfunction\sto\s"json_merge_patch()". +D 2017-03-22T21:45:20.759 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c ca27a98c0a7a90fcbaccd3157204e19afc7eec0ba3f4c08ed06bb638b773f523 +F ext/misc/json1.c be1032c54498ae8d884b7c430da876a0f895dfc734ac3255336014ce79e3f219 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -913,7 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 -F test/json104.test 83fd7a15eadb0cde34a37200842318d1cd98abe908e2847fb93d337d969815cc +F test/json104.test 66d3dfc9f76e413c7c957d897325b17b9d1bbdf471225310dbf47279e36f1937 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1568,10 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 -R 1d5a57aea0e07c508840a776caf26b14 -T *branch * json_mergepatch -T *sym-json_mergepatch * -T -sym-trunk * +P a267444039af519f088dd8f8ee33f686cc3071c087677075af2364ebc2587514 +R f0982f12ae9e098efe5bc17c4edc8338 U drh -Z 2961dcef6bde9e861663934a25c90d24 +Z 17fdf70c56513cf2488f301085ed16e1 diff --git a/manifest.uuid b/manifest.uuid index 980cac0acf..ea9e24d51d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a267444039af519f088dd8f8ee33f686cc3071c087677075af2364ebc2587514 \ No newline at end of file +53bf70f37bbca319ea35f70849e2a34ae628a504486158fdad5c4bb7431c68e0 \ No newline at end of file diff --git a/test/json104.test b/test/json104.test index fc2e0e38b4..be041c5da8 100644 --- a/test/json104.test +++ b/test/json104.test @@ -21,27 +21,24 @@ ifcapable !json1 { # This is the example from pages 2 and 3 of RFC-7396 do_execsql_test json104-100 { - SELECT json_mergepatch( - json('{ + SELECT json_merge_patch('{ "a": "b", "c": { "d": "e", "f": "g" } - }'), - json('{ + }','{ "a":"z", "c": { "f": null } - }')); + }'); } {{{"a":"z","c":{"d":"e"}}}} # This is the example from pages 4 and 5 of RFC-7396 do_execsql_test json104-110 { - SELECT json_mergepatch( - json('{ + SELECT json_merge_patch('{ "title": "Goodbye!", "author" : { "givenName" : "John", @@ -49,15 +46,14 @@ do_execsql_test json104-110 { }, "tags":[ "example", "sample" ], "content": "This will be unchanged" - }'), - json('{ + }','{ "title": "Hello!", "phoneNumber": "+01-123-456-7890", "author": { "familyName": null }, "tags": [ "example" ] - }')); + }'); } {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged",phoneNumber:"+01-123-456-7890"}}} finish_test From bb7aa2d85e684d0db13c93c1e6dbb0e2f3bfb1ab Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Mar 2017 00:13:52 +0000 Subject: [PATCH 284/292] Various fixes to the json_merge_patch() function. FossilOrigin-Name: f49fd2554b0723eb7cf2fd765d655c6820833ee7e5f7d7629d98c27a6fffa1d9 --- ext/misc/json1.c | 39 ++++++++++++++++++++++----------------- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json104.test | 9 ++++++++- 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index c037525c45..f03bdc6652 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1364,7 +1364,7 @@ static JsonNode *jsonMergePatch( JsonNode *pPatch /* The PATCH */ ){ int i, j; - int iApnd; + int iRoot; JsonNode *pTarget; if( pPatch->eType!=JSON_OBJECT ){ return pPatch; @@ -1373,14 +1373,14 @@ static JsonNode *jsonMergePatch( pTarget = &pParse->aNode[iTarget]; assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); if( pTarget->eType!=JSON_OBJECT ){ - for(i=2; in; i += jsonNodeSize(&pPatch[i])+1){ + for(i=2; i<=pPatch->n; i += jsonNodeSize(&pPatch[i])+1){ if( pPatch[i].eType==JSON_NULL ){ - pPatch[i-1].jnFlags |= JNODE_REMOVE; + pPatch[i].jnFlags |= JNODE_REMOVE; } } return pPatch; } - iApnd = iTarget; + iRoot = iTarget; for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ int nKey; const char *zKey; @@ -1388,21 +1388,19 @@ static JsonNode *jsonMergePatch( assert( pPatch[i].jnFlags & JNODE_LABEL ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; - if( (pPatch[i].jnFlags & JNODE_RAW)==0 ){ - assert( nKey>=2 && zKey[0]=='"' && zKey[nKey-1]=='"' ); - nKey -= 2; - zKey ++; - } + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); - if( jsonLabelCompare(&pTarget[j], zKey, nKey) - && (pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH))==0 + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( (pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH))==0 + && pTarget[j].n==nKey + && strncmp(pTarget[j].u.zJContent, zKey, nKey)==0 ){ if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; }else{ - JsonNode *pNew = jsonMergePatch(pParse, j+1, &pPatch[i+1]); + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); if( pNew==0 ) return 0; pTarget = &pParse->aNode[iTarget]; if( pNew!=&pTarget[j+1] ){ @@ -1413,16 +1411,16 @@ static JsonNode *jsonMergePatch( break; } } - if( j>=pTarget->n ){ + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ int iStart, iPatch; iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); if( pParse->oom ) return 0; pTarget = &pParse->aNode[iTarget]; - pParse->aNode[iApnd].jnFlags |= JNODE_APPEND; - pParse->aNode[iApnd].u.iAppend = iStart; - iApnd = iStart; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; } @@ -1442,13 +1440,20 @@ static void jsonMergePatchFunc( ){ JsonParse x; /* The JSON that is being patched */ JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ jsonParseReset(&x); return; } - jsonReturnJson(jsonMergePatch(&x, 0, y.aNode), ctx, 0); + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } jsonParseReset(&x); jsonParseReset(&y); } diff --git a/manifest b/manifest index 8c6f009d42..31881bdb35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\snew\sfunction\sto\s"json_merge_patch()". -D 2017-03-22T21:45:20.759 +C Various\sfixes\sto\sthe\sjson_merge_patch()\sfunction. +D 2017-03-23T00:13:52.903 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c be1032c54498ae8d884b7c430da876a0f895dfc734ac3255336014ce79e3f219 +F ext/misc/json1.c 8a8ba23a0715858ff393aa890e379912759712518afa4d28de6308a79287cb61 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -913,7 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 -F test/json104.test 66d3dfc9f76e413c7c957d897325b17b9d1bbdf471225310dbf47279e36f1937 +F test/json104.test 9f8358fd4ec94eca27b31d4ad7f75482edf24c865df378e5129c0b3107c9664e F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1568,7 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a267444039af519f088dd8f8ee33f686cc3071c087677075af2364ebc2587514 -R f0982f12ae9e098efe5bc17c4edc8338 +P 53bf70f37bbca319ea35f70849e2a34ae628a504486158fdad5c4bb7431c68e0 +R 7c38ec8ac02c81967ed35b7b062ffaf6 U drh -Z 17fdf70c56513cf2488f301085ed16e1 +Z e95ab9d86e57fcad88a27a5fb020f800 diff --git a/manifest.uuid b/manifest.uuid index ea9e24d51d..23d86ab688 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53bf70f37bbca319ea35f70849e2a34ae628a504486158fdad5c4bb7431c68e0 \ No newline at end of file +f49fd2554b0723eb7cf2fd765d655c6820833ee7e5f7d7629d98c27a6fffa1d9 \ No newline at end of file diff --git a/test/json104.test b/test/json104.test index be041c5da8..b5a45bc525 100644 --- a/test/json104.test +++ b/test/json104.test @@ -54,6 +54,13 @@ do_execsql_test json104-110 { }, "tags": [ "example" ] }'); -} {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged",phoneNumber:"+01-123-456-7890"}}} +} {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged","phoneNumber":"+01-123-456-7890"}}} + +do_execsql_test json104-200 { + SELECT json_merge_patch('[1,2,3]','{"x":null}'); +} {{{}}} +do_execsql_test json104-210 { + SELECT json_merge_patch('[1,2,3]','{"x":null,"y":1,"z":null}'); +} {{{"y":1}}} finish_test From 0002d246a94f03b7947c0fe4156f2dcc31cd722b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Mar 2017 00:46:15 +0000 Subject: [PATCH 285/292] Fix harmless compiler warnings in the new json_merge_patch() logic. FossilOrigin-Name: 5d2cf5a2f8afd88d041d89b3936042ce5a43629d23c48738cb2791b24da3e6af --- ext/misc/json1.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index f03bdc6652..31e03d1e7d 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1363,8 +1363,8 @@ static JsonNode *jsonMergePatch( int iTarget, /* Node of the TARGET in pParse */ JsonNode *pPatch /* The PATCH */ ){ - int i, j; - int iRoot; + u32 i, j; + u32 iRoot; JsonNode *pTarget; if( pPatch->eType!=JSON_OBJECT ){ return pPatch; diff --git a/manifest b/manifest index 31881bdb35..a21f45f4f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Various\sfixes\sto\sthe\sjson_merge_patch()\sfunction. -D 2017-03-23T00:13:52.903 +C Fix\sharmless\scompiler\swarnings\sin\sthe\snew\sjson_merge_patch()\slogic. +D 2017-03-23T00:46:15.319 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c 8a8ba23a0715858ff393aa890e379912759712518afa4d28de6308a79287cb61 +F ext/misc/json1.c bae7cfb49cda81be573fb159919b502a906940ccc1688dbaab33688489c9cf26 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -1568,7 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53bf70f37bbca319ea35f70849e2a34ae628a504486158fdad5c4bb7431c68e0 -R 7c38ec8ac02c81967ed35b7b062ffaf6 +P f49fd2554b0723eb7cf2fd765d655c6820833ee7e5f7d7629d98c27a6fffa1d9 +R b4e56917a91a0ee0d4fc1a8a4b3db16b U drh -Z e95ab9d86e57fcad88a27a5fb020f800 +Z 39d1ac0a1922cab73356f035d8da0baa diff --git a/manifest.uuid b/manifest.uuid index 23d86ab688..19e8a627a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49fd2554b0723eb7cf2fd765d655c6820833ee7e5f7d7629d98c27a6fffa1d9 \ No newline at end of file +5d2cf5a2f8afd88d041d89b3936042ce5a43629d23c48738cb2791b24da3e6af \ No newline at end of file From 1fe162fd0ce451dacee29c9d7164415c8815ca7b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Mar 2017 12:56:44 +0000 Subject: [PATCH 286/292] Avoid redundant edits in the json_merge_patch() function. FossilOrigin-Name: 4a8e6437dd610c5376a07867a73e5a7e2ea90357a018e1788ecce6f4e10bc939 --- ext/misc/json1.c | 6 ++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 31e03d1e7d..80b654d68d 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1393,10 +1393,8 @@ static JsonNode *jsonMergePatch( assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( (pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH))==0 - && pTarget[j].n==nKey - && strncmp(pTarget[j].u.zJContent, zKey, nKey)==0 - ){ + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; }else{ diff --git a/manifest b/manifest index a21f45f4f1..0c77a3ba53 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\snew\sjson_merge_patch()\slogic. -D 2017-03-23T00:46:15.319 +C Avoid\sredundant\sedits\sin\sthe\sjson_merge_patch()\sfunction. +D 2017-03-23T12:56:44.298 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c bae7cfb49cda81be573fb159919b502a906940ccc1688dbaab33688489c9cf26 +F ext/misc/json1.c dbb168b0a7640a896a565c647dcabb7c7d116c9d2b1a23b963b1d177edc0ddba F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -1568,7 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f49fd2554b0723eb7cf2fd765d655c6820833ee7e5f7d7629d98c27a6fffa1d9 -R b4e56917a91a0ee0d4fc1a8a4b3db16b +P 5d2cf5a2f8afd88d041d89b3936042ce5a43629d23c48738cb2791b24da3e6af +R 76e620c002fc08a86767847f533fe70a U drh -Z 39d1ac0a1922cab73356f035d8da0baa +Z 538e213b7e95dcafcfcd2e7dd4b1a792 diff --git a/manifest.uuid b/manifest.uuid index 19e8a627a7..da2d4e0201 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d2cf5a2f8afd88d041d89b3936042ce5a43629d23c48738cb2791b24da3e6af \ No newline at end of file +4a8e6437dd610c5376a07867a73e5a7e2ea90357a018e1788ecce6f4e10bc939 \ No newline at end of file From a77edc69eebe610d309f62234f4d5b277a80102a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Mar 2017 17:03:35 +0000 Subject: [PATCH 287/292] Remove an invalid assert() statement failing when VACUUMing a database that contains an index on a column explicitly declared "COLLATE BINARY". FossilOrigin-Name: 9f2e04d3c3526b5ff60d941aa6d5446f602cab37cb93972937f39eefabd6868d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 2 -- test/collateB.test | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 560760071c..a838663224 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\sshell. -D 2017-03-22T12:51:34.759 +C Remove\san\sinvalid\sassert()\sstatement\sfailing\swhen\sVACUUMing\sa\sdatabase\sthat\ncontains\san\sindex\son\sa\scolumn\sexplicitly\sdeclared\s"COLLATE\sBINARY". +D 2017-03-23T17:03:35.684 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -363,7 +363,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 3ed64afc49c0a2221e397b9f65d231ffbef506fe +F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a68d8d1d14cf7488bb29dc5311cb1ce9a4404258 F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760 @@ -597,7 +597,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/collateB.test 8ec2accd2d7166c1eff0d2a39bc90262c6f89632 +F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 055b36f1c1593bb123f7319a07c476143d71af052b5b8d34afcd0d500f197882 -R 1f280e53b7c950d24565e873ae0276ec -U drh -Z a154993c25ee99ce8fe87b6f82cb272d +P a786829783ef65ef270ca360712364cbd47a540d31ed1547d50808aad698bea7 +R 85624f613e7e3af447c83f8947d69a8d +U dan +Z 3c9d547950272a881cd381bfe4f2d9b9 diff --git a/manifest.uuid b/manifest.uuid index 6c408aa251..1f6f43e2c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a786829783ef65ef270ca360712364cbd47a540d31ed1547d50808aad698bea7 \ No newline at end of file +9f2e04d3c3526b5ff60d941aa6d5446f602cab37cb93972937f39eefabd6868d \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index f1f3807244..a1f5442515 100644 --- a/src/insert.c +++ b/src/insert.c @@ -2227,8 +2227,6 @@ static int xferOptimization( ** sorted order. */ for(i=0; inColumn; i++){ const char *zColl = pSrcIdx->azColl[i]; - assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 - || sqlite3StrBINARY==zColl ); if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ diff --git a/test/collateB.test b/test/collateB.test index 4815de83ee..3c8d50d8e6 100644 --- a/test/collateB.test +++ b/test/collateB.test @@ -13,6 +13,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix collateB do_execsql_test collateB-1.1 { CREATE TABLE t1(a INTEGER PRIMARY KEY); @@ -60,4 +61,17 @@ do_execsql_test collateB-1.17 { SELECT *,'|' FROM t1, t2, t3 WHERE b=x2 AND a=x1 AND 1=a; } {1 11 1 11 |} +#------------------------------------------------------------------------- +# Test an assert() failure that was occuring if an index were created +# on a column explicitly declared "COLLATE binary". +reset_db +do_execsql_test 2.1 { + CREATE TABLE t4(a COLLATE binary); + CREATE INDEX i4 ON t4(a); + INSERT INTO t4 VALUES('one'), ('two'), ('three'); + VACUUM; +} + +integrity_check 2.2 + finish_test From 8fe3e9d882c8e667ac20e3b090687b1fc5c23003 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Mar 2017 17:22:09 +0000 Subject: [PATCH 288/292] Do not run test file autoanalyze1.test if OMIT_VIRTUALTABLE is defined. FossilOrigin-Name: e1d06a573ee56bf6a14e4dc8086d6a4df1464f2914834cf4596a0a406688f608 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/autoanalyze1.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a838663224..24dc30b166 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sinvalid\sassert()\sstatement\sfailing\swhen\sVACUUMing\sa\sdatabase\sthat\ncontains\san\sindex\son\sa\scolumn\sexplicitly\sdeclared\s"COLLATE\sBINARY". -D 2017-03-23T17:03:35.684 +C Do\snot\srun\stest\sfile\sautoanalyze1.test\sif\sOMIT_VIRTUALTABLE\sis\sdefined. +D 2017-03-23T17:22:09.188 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -528,7 +528,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f -F test/autoanalyze1.test 1ba80d5e1412d46503566b70741a5eea060da929 +F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a786829783ef65ef270ca360712364cbd47a540d31ed1547d50808aad698bea7 -R 85624f613e7e3af447c83f8947d69a8d +P 9f2e04d3c3526b5ff60d941aa6d5446f602cab37cb93972937f39eefabd6868d +R c389878b6e059490204890f1ea916f73 U dan -Z 3c9d547950272a881cd381bfe4f2d9b9 +Z b2d2e5c74dc6093684311a4bed08a5d2 diff --git a/manifest.uuid b/manifest.uuid index 1f6f43e2c5..aed4b7d66c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f2e04d3c3526b5ff60d941aa6d5446f602cab37cb93972937f39eefabd6868d \ No newline at end of file +e1d06a573ee56bf6a14e4dc8086d6a4df1464f2914834cf4596a0a406688f608 \ No newline at end of file diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test index 77e767a89e..71e5d6c163 100644 --- a/test/autoanalyze1.test +++ b/test/autoanalyze1.test @@ -29,7 +29,7 @@ source $testdir/tester.tcl # These tests also use "PRAGMA stats" which are only enabled for # debugging builds. # -ifcapable {!debug || !analyze} { +ifcapable {!debug || !analyze || !vtab} { finish_test return } From db528529bf9b1524c5d31c52ee04aaaa6466b0e5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Mar 2017 19:51:38 +0000 Subject: [PATCH 289/292] Improvement to the amalgamation configure.ac file contributed by Bob Friesenhahn. FossilOrigin-Name: bf28a55d8185e370fea39de297c37387f852f43fac0a316e5900e633f99b367f --- autoconf/configure.ac | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/autoconf/configure.ac b/autoconf/configure.ac index b9a11aac8f..5a607de054 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -55,9 +55,9 @@ AS_IF([ test x"$enable_editline" != xno ],[ LIBS="" AC_SEARCH_LIBS([readline],[edit],[ AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) - READLINE_LIBS=$LIBS + READLINE_LIBS="$LIBS -ltinfo" enable_readline=no - ]) + ],[],[-ltinfo]) AS_UNSET(ac_cv_search_readline) LIBS=$sLIBS ]) diff --git a/manifest b/manifest index 24dc30b166..c5b123ea51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\stest\sfile\sautoanalyze1.test\sif\sOMIT_VIRTUALTABLE\sis\sdefined. -D 2017-03-23T17:22:09.188 +C Improvement\sto\sthe\samalgamation\sconfigure.ac\sfile\scontributed\sby\nBob\sFriesenhahn. +D 2017-03-23T19:51:38.579 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -14,7 +14,7 @@ F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 F autoconf/Makefile.msc 92e3d7fd64dc5c23a23d33824ee4709aaea00a6331c4bb41a1aadcf6600a49f7 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc +F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -1567,7 +1567,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f2e04d3c3526b5ff60d941aa6d5446f602cab37cb93972937f39eefabd6868d -R c389878b6e059490204890f1ea916f73 -U dan -Z b2d2e5c74dc6093684311a4bed08a5d2 +P e1d06a573ee56bf6a14e4dc8086d6a4df1464f2914834cf4596a0a406688f608 +R 05b216864be03a3eba3ef1e51323e9ca +U drh +Z 5c18c26b109b1904b072f9a45f7157f5 diff --git a/manifest.uuid b/manifest.uuid index aed4b7d66c..d40ca490c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1d06a573ee56bf6a14e4dc8086d6a4df1464f2914834cf4596a0a406688f608 \ No newline at end of file +bf28a55d8185e370fea39de297c37387f852f43fac0a316e5900e633f99b367f \ No newline at end of file From 37f03dfb1f1f0c5668ef9d8ab61352712d368c96 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Mar 2017 20:33:49 +0000 Subject: [PATCH 290/292] Change the name of the json_merge_patch() function to just json_patch(). FossilOrigin-Name: 04d4100445a3373986ee962618bc03ec304f6ba2f867c8e9eee415daffb593fc --- ext/misc/json1.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json104.test | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 80b654d68d..acec87c1b8 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1431,7 +1431,7 @@ static JsonNode *jsonMergePatch( ** object that is the result of running the RFC 7396 MergePatch() algorithm ** on the two arguments. */ -static void jsonMergePatchFunc( +static void jsonPatchFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv @@ -2260,8 +2260,8 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_array_length", 2, 0, jsonArrayLengthFunc }, { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, - { "json_merge_patch", 2, 0, jsonMergePatchFunc }, { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, { "json_replace", -1, 0, jsonReplaceFunc }, diff --git a/manifest b/manifest index 0c77a3ba53..d5d959a6f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredundant\sedits\sin\sthe\sjson_merge_patch()\sfunction. -D 2017-03-23T12:56:44.298 +C Change\sthe\sname\sof\sthe\sjson_merge_patch()\sfunction\sto\sjust\sjson_patch(). +D 2017-03-23T20:33:49.228 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c dbb168b0a7640a896a565c647dcabb7c7d116c9d2b1a23b963b1d177edc0ddba +F ext/misc/json1.c c065914b126b5f4696b824286fca153da8035bb8c660a15193ea2711668f1d14 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -913,7 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 -F test/json104.test 9f8358fd4ec94eca27b31d4ad7f75482edf24c865df378e5129c0b3107c9664e +F test/json104.test b6808a3efb973235331c274ebe251bfbf75c6fff0ff12956fb9719cc227a9f5b F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1568,7 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d2cf5a2f8afd88d041d89b3936042ce5a43629d23c48738cb2791b24da3e6af -R 76e620c002fc08a86767847f533fe70a +P 4a8e6437dd610c5376a07867a73e5a7e2ea90357a018e1788ecce6f4e10bc939 +R d0e9240b8b29f168e87723d2f33d4c1f U drh -Z 538e213b7e95dcafcfcd2e7dd4b1a792 +Z bc4bcd04349b5b8bc6712e6bdee5b622 diff --git a/manifest.uuid b/manifest.uuid index da2d4e0201..7097a2526e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a8e6437dd610c5376a07867a73e5a7e2ea90357a018e1788ecce6f4e10bc939 \ No newline at end of file +04d4100445a3373986ee962618bc03ec304f6ba2f867c8e9eee415daffb593fc \ No newline at end of file diff --git a/test/json104.test b/test/json104.test index b5a45bc525..ff17147723 100644 --- a/test/json104.test +++ b/test/json104.test @@ -8,7 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# This file implements tests for json_mergepatch(A,B) SQL function. +# This file implements tests for json_patch(A,B) SQL function. # set testdir [file dirname $argv0] @@ -21,7 +21,7 @@ ifcapable !json1 { # This is the example from pages 2 and 3 of RFC-7396 do_execsql_test json104-100 { - SELECT json_merge_patch('{ + SELECT json_patch('{ "a": "b", "c": { "d": "e", @@ -38,7 +38,7 @@ do_execsql_test json104-100 { # This is the example from pages 4 and 5 of RFC-7396 do_execsql_test json104-110 { - SELECT json_merge_patch('{ + SELECT json_patch('{ "title": "Goodbye!", "author" : { "givenName" : "John", @@ -57,10 +57,10 @@ do_execsql_test json104-110 { } {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged","phoneNumber":"+01-123-456-7890"}}} do_execsql_test json104-200 { - SELECT json_merge_patch('[1,2,3]','{"x":null}'); + SELECT json_patch('[1,2,3]','{"x":null}'); } {{{}}} do_execsql_test json104-210 { - SELECT json_merge_patch('[1,2,3]','{"x":null,"y":1,"z":null}'); + SELECT json_patch('[1,2,3]','{"x":null,"y":1,"z":null}'); } {{{"y":1}}} finish_test From 29c996987114728518feb15903b5366c00023164 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Mar 2017 12:35:17 +0000 Subject: [PATCH 291/292] Fix an error in the new json_patch() routine discovered by Ralf Junker. FossilOrigin-Name: 9d5350418b2f6113e0b50c57e8a872006f27753067baf08ffdfa7943c0c9a148 --- ext/misc/json1.c | 26 +++++++++++++++++++++----- manifest | 15 +++++++-------- manifest.uuid | 2 +- test/json104.test | 10 ++++++++++ 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index acec87c1b8..2689726302 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1164,6 +1164,25 @@ static void jsonWrongNumArgs( sqlite3_free(zMsg); } +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + /**************************************************************************** ** SQL functions used for testing and debugging @@ -1373,11 +1392,7 @@ static JsonNode *jsonMergePatch( pTarget = &pParse->aNode[iTarget]; assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); if( pTarget->eType!=JSON_OBJECT ){ - for(i=2; i<=pPatch->n; i += jsonNodeSize(&pPatch[i])+1){ - if( pPatch[i].eType==JSON_NULL ){ - pPatch[i].jnFlags |= JNODE_REMOVE; - } - } + jsonRemoveAllNulls(pPatch); return pPatch; } iRoot = iTarget; @@ -1415,6 +1430,7 @@ static JsonNode *jsonMergePatch( jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); pTarget = &pParse->aNode[iTarget]; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; pParse->aNode[iRoot].u.iAppend = iStart - iRoot; diff --git a/manifest b/manifest index b843ff3c58..b1f0b85dcc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sjson_patch()\sSQL\sfunction\sto\sthe\sJSON1\sextension. -D 2017-03-23T23:44:55.894 +C Fix\san\serror\sin\sthe\snew\sjson_patch()\sroutine\sdiscovered\sby\sRalf\sJunker. +D 2017-03-24T12:35:17.534 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -218,7 +218,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c c065914b126b5f4696b824286fca153da8035bb8c660a15193ea2711668f1d14 +F ext/misc/json1.c 52f93b91b8bbbfcd84dca34be383974137e83a1673c71d5f728b16b68bb57d9c F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e @@ -913,7 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 -F test/json104.test b6808a3efb973235331c274ebe251bfbf75c6fff0ff12956fb9719cc227a9f5b +F test/json104.test 2966101e14463655546eb8f639db25382957e9b7136de717e17db945a6f539fb F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1568,8 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf28a55d8185e370fea39de297c37387f852f43fac0a316e5900e633f99b367f 04d4100445a3373986ee962618bc03ec304f6ba2f867c8e9eee415daffb593fc -R 114c4ffb4ff05c31b0d61e7341b119eb -T +closed 04d4100445a3373986ee962618bc03ec304f6ba2f867c8e9eee415daffb593fc +P 476088482024e411e2549b1697cdaf0124294c79d43f508c71c4eb66906a56fc +R 08d6f399deb89f284d166c511d46a6ea U drh -Z 015dcf3dd857377b8513b610e324274e +Z 437a93d0f27334776c018af18840b42a diff --git a/manifest.uuid b/manifest.uuid index 4f00615e01..4ff617a710 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -476088482024e411e2549b1697cdaf0124294c79d43f508c71c4eb66906a56fc \ No newline at end of file +9d5350418b2f6113e0b50c57e8a872006f27753067baf08ffdfa7943c0c9a148 \ No newline at end of file diff --git a/test/json104.test b/test/json104.test index ff17147723..5d93991eb1 100644 --- a/test/json104.test +++ b/test/json104.test @@ -62,5 +62,15 @@ do_execsql_test json104-200 { do_execsql_test json104-210 { SELECT json_patch('[1,2,3]','{"x":null,"y":1,"z":null}'); } {{{"y":1}}} +do_execsql_test json104-220 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}'); +} {{{"a":{"bb":{}}}}} +do_execsql_test json104-221 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,null,3]}}}'); +} {{{"a":{"bb":{"ccc":[1,null,3]}}}}} +do_execsql_test json104-222 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}'); +} {{{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}}} + finish_test From f9e91972c94e198a5c6406eeccb7068c35779211 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Mar 2017 13:31:47 +0000 Subject: [PATCH 292/292] Add the RFC-7396 Appendix A test cases for json_patch(). FossilOrigin-Name: c5441d2df2526723f72610cc14dd243223663979e67ecdd76fe06fcd366f2b29 --- manifest | 12 +++++------ manifest.uuid | 2 +- test/json104.test | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b1f0b85dcc..cfdc624046 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\snew\sjson_patch()\sroutine\sdiscovered\sby\sRalf\sJunker. -D 2017-03-24T12:35:17.534 +C Add\sthe\sRFC-7396\sAppendix\sA\stest\scases\sfor\sjson_patch(). +D 2017-03-24T13:31:47.314 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1faf9f06aadc9284c212dea7bbc7c0dea7e8337f0287c81001eff500912c790a @@ -913,7 +913,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 -F test/json104.test 2966101e14463655546eb8f639db25382957e9b7136de717e17db945a6f539fb +F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1568,7 +1568,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 476088482024e411e2549b1697cdaf0124294c79d43f508c71c4eb66906a56fc -R 08d6f399deb89f284d166c511d46a6ea +P 9d5350418b2f6113e0b50c57e8a872006f27753067baf08ffdfa7943c0c9a148 +R f5d3be53ea90768240f7c6616a0ecd20 U drh -Z 437a93d0f27334776c018af18840b42a +Z c6323f1875b7921e440a4d008ab9c650 diff --git a/manifest.uuid b/manifest.uuid index 4ff617a710..40296ca7c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d5350418b2f6113e0b50c57e8a872006f27753067baf08ffdfa7943c0c9a148 \ No newline at end of file +c5441d2df2526723f72610cc14dd243223663979e67ecdd76fe06fcd366f2b29 \ No newline at end of file diff --git a/test/json104.test b/test/json104.test index 5d93991eb1..b5313f01b5 100644 --- a/test/json104.test +++ b/test/json104.test @@ -72,5 +72,59 @@ do_execsql_test json104-222 { SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}'); } {{{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}}} +# Example test cases at the end of the RFC-7396 document +do_execsql_test json104-300 { + SELECT json_patch('{"a":"b"}','{"a":"c"}'); +} {{{"a":"c"}}} +do_execsql_test json104-300a { + SELECT coalesce(json_patch(null,'{"a":"c"}'), 'real-null'); +} {{real-null}} +do_execsql_test json104-301 { + SELECT json_patch('{"a":"b"}','{"b":"c"}'); +} {{{"a":"b","b":"c"}}} +do_execsql_test json104-302 { + SELECT json_patch('{"a":"b"}','{"a":null}'); +} {{{}}} +do_execsql_test json104-303 { + SELECT json_patch('{"a":"b","b":"c"}','{"a":null}'); +} {{{"b":"c"}}} +do_execsql_test json104-304 { + SELECT json_patch('{"a":["b"]}','{"a":"c"}'); +} {{{"a":"c"}}} +do_execsql_test json104-305 { + SELECT json_patch('{"a":"c"}','{"a":["b"]}'); +} {{{"a":["b"]}}} +do_execsql_test json104-306 { + SELECT json_patch('{"a":{"b":"c"}}','{"a":{"b":"d","c":null}}'); +} {{{"a":{"b":"d"}}}} +do_execsql_test json104-307 { + SELECT json_patch('{"a":[{"b":"c"}]}','{"a":[1]}'); +} {{{"a":[1]}}} +do_execsql_test json104-308 { + SELECT json_patch('["a","b"]','["c","d"]'); +} {{["c","d"]}} +do_execsql_test json104-309 { + SELECT json_patch('{"a":"b"}','["c"]'); +} {{["c"]}} +do_execsql_test json104-310 { + SELECT json_patch('{"a":"foo"}','null'); +} {{null}} +do_execsql_test json104-310a { + SELECT coalesce(json_patch('{"a":"foo"}',null), 'real-null'); +} {{real-null}} +do_execsql_test json104-311 { + SELECT json_patch('{"a":"foo"}','"bar"'); +} {{"bar"}} +do_execsql_test json104-312 { + SELECT json_patch('{"e":null}','{"a":1}'); +} {{{"e":null,"a":1}}} +do_execsql_test json104-313 { + SELECT json_patch('[1,2]','{"a":"b","c":null}'); +} {{{"a":"b"}}} +do_execsql_test json104-314 { + SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}'); +} {{{"a":{"bb":{}}}}} + + finish_test