From 21f2bafd9ba4cba4b6a837166cc673424dabe3f1 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Sep 2017 07:46:54 +0000 Subject: [PATCH 001/156] Experimental change so that snapshot transactions always lock the wal file - preventing writers or truncate-checkpointers from wrapping it. FossilOrigin-Name: d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1 --- manifest | 18 ++++---- manifest.uuid | 2 +- src/wal.c | 3 +- test/snapshot3.test | 100 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 test/snapshot3.test diff --git a/manifest b/manifest index 1938515870..c550d72b06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\supdated\sWin32\sVFS\ssemantics\sfor\swinOpen\sfrom\scheck-in\s[5d03c738e9]\sfor\sWinRT,\set\sal,\sas\swell. -D 2017-09-22T16:23:23.946 +C Experimental\schange\sso\sthat\ssnapshot\stransactions\salways\slock\sthe\swal\sfile\s-\npreventing\swriters\sor\struncate-checkpointers\sfrom\swrapping\sit. +D 2017-09-23T07:46:54.932 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a +F src/wal.c 0ceefa9f2a0cb00033d312a04c2d5c2005a6404a6fccab22743de9b3c363154b F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 21eea981920a13fd3c0ac3d6c128d0a34b22cbec064e4f0603375fe1ffe26ca6 @@ -1206,6 +1206,7 @@ F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f F test/snapshot2.test 867652ed4a13282dce218723535fad1c7b44c3c4 +F test/snapshot3.test 9719443594a04778861bd20d12596c5f880af177d6cd62f111da3198cafc6096 F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1655,7 +1656,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355 -R 7f08f0df04c939d1dd5a187c9ae3cc49 -U mistachkin -Z 321f82ff3326ba269549cfe661409f34 +P 2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98 +R 44e3d692cd4dca7384d5a55f548b9b05 +T *branch * snapshots-lock-wal +T *sym-snapshots-lock-wal * +T -sym-trunk * +U dan +Z 91f0de7cf02d66dc9c74cc874f9821f7 diff --git a/manifest.uuid b/manifest.uuid index 04153e1aac..b33987e64e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98 \ No newline at end of file +d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 9930b84421..73a6db0e02 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2250,8 +2250,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ pInfo = walCkptInfo(pWal); if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT - && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 - || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) #endif ){ /* The WAL has been completely backfilled (or it is empty). diff --git a/test/snapshot3.test b/test/snapshot3.test new file mode 100644 index 0000000000..266f43dfd4 --- /dev/null +++ b/test/snapshot3.test @@ -0,0 +1,100 @@ +# 2016 September 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The focus +# of this file is the sqlite3_snapshot_xxx() APIs. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !snapshot {finish_test; return} +set testprefix snapshot3 + +# This test does not work with the inmemory_journal permutation. The reason +# is that each connection opened as part of this permutation executes +# "PRAGMA journal_mode=memory", which fails if the database is in wal mode +# and there are one or more existing connections. +if {[permutation]=="inmemory_journal"} { + finish_test + return +} + +#------------------------------------------------------------------------- +# This block of tests verifies that it is not possible to wrap the wal +# file - using a writer or a "PRAGMA wal_checkpoint = TRUNCATE" - while +# there is an open snapshot transaction (transaction opened using +# sqlite3_snapshot_open()). +# +do_execsql_test 1.0 { + CREATE TABLE t1(y); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); +} {wal} + +do_test 1.1 { + sqlite3 db2 test.db + sqlite3 db3 test.db + + execsql {SELECT * FROM sqlite_master} db2 + execsql {SELECT * FROM sqlite_master} db3 + + db2 trans { set snap [sqlite3_snapshot_get_blob db2 main] } + db2 eval { SELECT * FROM t1 } +} {1 2 3 4} + +do_test 1.2 { + execsql BEGIN db2 + sqlite3_snapshot_open_blob db2 main $snap + db2 eval { SELECT * FROM t1 } +} {1 2 3 4} + +do_test 1.2 { + execsql END db2 + execsql { PRAGMA wal_checkpoint } + + execsql BEGIN db2 + sqlite3_snapshot_open_blob db2 main $snap + db2 eval { SELECT * FROM t1 } +} {1 2 3 4} + +set sz [file size test.db-wal] +do_test 1.3 { + execsql { PRAGMA wal_checkpoint = truncate } + file size test.db-wal +} $sz + +do_test 1.4 { + execsql BEGIN db3 + list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg +} {0 {}} + +do_test 1.5 { + db3 eval { SELECT * FROM t1; END } +} {1 2 3 4} + +do_test 1.6 { + db2 eval { SELECT * FROM t1; END } +} {1 2 3 4} + +do_test 1.7 { + execsql { PRAGMA wal_checkpoint = truncate } + file size test.db-wal +} 0 + +do_test 1.8 { + execsql BEGIN db3 + list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg +} {1 SQLITE_BUSY_SNAPSHOT} + +finish_test + From a43c8c8a6065b3eed4b1a7ec03a7535f3b3b537f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 13:48:11 +0000 Subject: [PATCH 002/156] Initial implementation of the "sqlite_dbpage" virtual table. Currently it is read-only and has a place-holder xBestIndex. FossilOrigin-Name: c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 --- Makefile.in | 8 +- Makefile.msc | 9 +- main.mk | 5 +- manifest | 26 +++-- manifest.uuid | 2 +- src/dbpage.c | 241 ++++++++++++++++++++++++++++++++++++++++++++ src/main.c | 6 ++ src/sqliteInt.h | 3 + tool/mksqlite3c.tcl | 1 + 9 files changed, 285 insertions(+), 16 deletions(-) create mode 100644 src/dbpage.c diff --git a/Makefile.in b/Makefile.in index 58c2c2919b..7ea97e6195 100644 --- a/Makefile.in +++ b/Makefile.in @@ -166,7 +166,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@ # LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ + callback.lo complete.lo ctime.lo \ + date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ @@ -215,6 +216,7 @@ SRC = \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -449,6 +451,7 @@ TESTSRC2 = \ $(TOP)/src/build.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ @@ -752,6 +755,9 @@ ctime.lo: $(TOP)/src/ctime.c $(HDR) date.lo: $(TOP)/src/date.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c +dbpage.lo: $(TOP)/src/dbpage.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c + dbstat.lo: $(TOP)/src/dbstat.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c diff --git a/Makefile.msc b/Makefile.msc index 10585e4a58..d59997388d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1091,7 +1091,8 @@ LTLIBS = $(LTLIBS) $(LIBICU) # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ + callback.lo complete.lo ctime.lo \ + date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ @@ -1154,6 +1155,7 @@ SRC00 = \ $(TOP)\src\complete.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ + $(TOP)\src\dbpage.c \ $(TOP)\src\dbstat.c \ $(TOP)\src\delete.c \ $(TOP)\src\expr.c \ @@ -1747,7 +1749,10 @@ ctime.lo: $(TOP)\src\ctime.c $(HDR) date.lo: $(TOP)\src\date.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c -dbstat.lo: $(TOP)\src\date.c $(HDR) +dbpage.lo: $(TOP)\src\dbpage.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c + +dbstat.lo: $(TOP)\src\dbstat.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c delete.lo: $(TOP)\src\delete.c $(HDR) diff --git a/main.mk b/main.mk index 7da6db15f1..3f83197d4d 100644 --- a/main.mk +++ b/main.mk @@ -55,7 +55,8 @@ THREADLIB += $(LIBS) LIBOBJ+= vdbe.o parse.o \ alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o \ + callback.o complete.o ctime.o \ + date.o dbpage.o dbstat.o delete.o expr.o \ fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ @@ -96,6 +97,7 @@ SRC = \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -359,6 +361,7 @@ TESTSRC2 = \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ + $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ diff --git a/manifest b/manifest index b8e6ca811e..dbf72466e4 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C In\sthe\sspeed-check.sh\stest\sscript,\sallow\san\sadditional\stest-name\sargument\nwhich\sbecomes\sthe\scomparison\sbaseline,\sin\splace\sof\s"trunk". -D 2017-10-11T12:20:36.082 -F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff +C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex. +D 2017-10-11T13:48:11.877 +F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f +F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222 +F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -409,6 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 +F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a @@ -423,7 +424,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c a4bdadaaa827e7380cba4de878ed7947dab5aeb84f617118ba6a0422cd745b4b +F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -464,7 +465,7 @@ F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h c07bc88eca1f59ce73e1f486187d0df4effe67c4579e112dfdd91c159e5c0569 +F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1597,7 +1598,7 @@ F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb -F tool/mksqlite3c.tcl b258d679829a9305f5cf107b7d97b9bf23adb3773df42947fed5ef7b180dfbd9 +F tool/mksqlite3c.tcl 1fb69d39166f52d802a70ec37d99bca51d011c8ab30be27bc495be493196ae41 F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 @@ -1656,7 +1657,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32 -R 9a03aa36b167c86705816bfaa31cdc19 +P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 +R 9d91f38d6de2e0aabd371a2c4dbcd832 +T *branch * dbpage +T *sym-dbpage * +T -sym-trunk * U drh -Z 8f5e2f1e221681e7d4bf20eb32b86754 +Z 95e75b385006fb35e51e104eb16c88ba diff --git a/manifest.uuid b/manifest.uuid index 7305d2515a..b0152fffa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 \ No newline at end of file +c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c new file mode 100644 index 0000000000..89dcf04227 --- /dev/null +++ b/src/dbpage.c @@ -0,0 +1,241 @@ +/* +** 2017-10-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_dbpage" virtual table. +** +** The sqlite_dbpage virtual table is used to read or write whole raw +** pages of the database file. The pager interface is used so that +** uncommitted changes and changes recorded in the WAL file are correctly +** retrieved. +*/ + +#include "sqliteInt.h" /* Requires access to internal data structures */ +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + +typedef struct DbpageTable DbpageTable; +typedef struct DbpageCursor DbpageCursor; + +struct DbpageCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int pgno; /* Current page number */ +}; + +struct DbpageTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database */ + Pager *pPager; /* Pager being read/written */ + int iDb; /* Index of database to analyze */ + int szPage; /* Size of each page in bytes */ + int nPage; /* Number of pages in the file */ +}; + +/* +** Connect to or create a dbpagevfs virtual table. +*/ +static int dbpageConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbpageTable *pTab = 0; + int rc = SQLITE_OK; + int iDb; + + if( argc>=4 ){ + Token nm; + sqlite3TokenInit(&nm, (char*)argv[3]); + iDb = sqlite3FindDb(db, &nm); + if( iDb<0 ){ + *pzErr = sqlite3_mprintf("no such schema: %s", argv[3]); + return SQLITE_ERROR; + } + }else{ + iDb = 0; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)"); + if( rc==SQLITE_OK ){ + pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + Btree *pBt = db->aDb[iDb].pBt; + memset(pTab, 0, sizeof(DbpageTable)); + pTab->db = db; + pTab->iDb = iDb; + pTab->pPager = pBt ? sqlite3BtreePager(pBt) : 0; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbpagevfs virtual table. +*/ +static int dbpageDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 full table scan +** 1 pgno=?1 +*/ +static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + return SQLITE_OK; +} + +/* +** Open a new dbpagevfs cursor. +*/ +static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbpageCursor *pCsr; + + pCsr = (DbpageCursor *)sqlite3_malloc64(sizeof(DbpageCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memset(pCsr, 0, sizeof(DbpageCursor)); + pCsr->base.pVtab = pVTab; + pCsr->pgno = -1; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a dbpagevfs cursor. +*/ +static int dbpageClose(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a dbpagevfs cursor to the next entry in the file. +*/ +static int dbpageNext(sqlite3_vtab_cursor *pCursor){ + int rc = SQLITE_OK; + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + pCsr->pgno++; + return rc; +} + +static int dbpageEof(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + return pCsr->pgno >= pTab->nPage; +} + +static int dbpageFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc = SQLITE_OK; + Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + + if( idxNum==1 ){ + pCsr->pgno = sqlite3_value_int(argv[0]); + }else{ + pCsr->pgno = 0; + } + pTab->szPage = sqlite3BtreeGetPageSize(pBt); + pTab->nPage = sqlite3BtreeLastPage(pBt); + return rc; +} + +static int dbpageColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc = SQLITE_OK; + switch( i ){ + case 0: { /* pgno */ + sqlite3_result_int(ctx, pCsr->pgno); + break; + } + case 1: { /* data */ + DbPage *pDbPage = 0; + rc = sqlite3PagerGet(pTab->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pTab->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); + break; + } + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + sqlite3_result_text(ctx, db->aDb[pTab->iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + *pRowid = pCsr->pgno; + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "dbpage" virtual table module +*/ +int sqlite3DbpageRegister(sqlite3 *db){ + static sqlite3_module dbpage_module = { + 0, /* iVersion */ + dbpageConnect, /* xCreate */ + dbpageConnect, /* xConnect */ + dbpageBestIndex, /* xBestIndex */ + dbpageDisconnect, /* xDisconnect */ + dbpageDisconnect, /* xDestroy */ + dbpageOpen, /* xOpen - open a cursor */ + dbpageClose, /* xClose - close a cursor */ + dbpageFilter, /* xFilter - configure scan constraints */ + dbpageNext, /* xNext - advance a cursor */ + dbpageEof, /* xEof - check for end of scan */ + dbpageColumn, /* xColumn - read data */ + dbpageRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + }; + return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); +} +#elif defined(SQLITE_ENABLE_DBPAGE_VTAB) +int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ diff --git a/src/main.c b/src/main.c index 3d7609ce5b..49613f6c74 100644 --- a/src/main.c +++ b/src/main.c @@ -3054,6 +3054,12 @@ static int openDatabase( } #endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbpageRegister(db); + } +#endif + #ifdef SQLITE_ENABLE_DBSTAT_VTAB if( !db->mallocFailed && rc==SQLITE_OK){ rc = sqlite3DbstatRegister(db); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a8f1bed512..0cc435d7b7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4400,6 +4400,9 @@ int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); #endif +#if defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST) +int sqlite3DbpageRegister(sqlite3*); +#endif #if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) int sqlite3DbstatRegister(sqlite3*); #endif diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 933819d12b..8ea3e81c91 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -394,6 +394,7 @@ foreach file { fts3_icu.c sqlite3rbu.c dbstat.c + dbpage.c sqlite3session.c json1.c fts5.c From 34d0b1ac56e7cdf587dfbb06a136d560032deedf Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 15:02:53 +0000 Subject: [PATCH 003/156] Get writes working on the sqlite_dbpage virtual table. Add a few test cases. FossilOrigin-Name: a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f --- manifest | 16 ++++--- manifest.uuid | 2 +- src/dbpage.c | 106 +++++++++++++++++++++++++++++++++++++++++++---- test/dbpage.test | 69 ++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 19 deletions(-) create mode 100644 test/dbpage.test diff --git a/manifest b/manifest index dbf72466e4..3e606d8851 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex. -D 2017-10-11T13:48:11.877 +C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases. +D 2017-10-11T15:02:53.047 F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d @@ -409,7 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5 +F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a @@ -709,6 +709,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e +F test/dbpage.test 9cf4dc92a4de67c81e5c32b24e3fbb8c4757e4b642694a219b3090a4f9277a4d F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1657,10 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49 -R 9d91f38d6de2e0aabd371a2c4dbcd832 -T *branch * dbpage -T *sym-dbpage * -T -sym-trunk * +P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 +R 92490bca397034b7f3fba4e951f0781d U drh -Z 95e75b385006fb35e51e104eb16c88ba +Z ddf133cf759c24442aeeb91134927fef diff --git a/manifest.uuid b/manifest.uuid index b0152fffa9..7a388fce6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 \ No newline at end of file +a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 89dcf04227..d21c5b6df1 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -16,6 +16,19 @@ ** pages of the database file. The pager interface is used so that ** uncommitted changes and changes recorded in the WAL file are correctly ** retrieved. +** +** Usage example: +** +** SELECT data FROM sqlite_dbpage('aux1') WHERE pgno=123; +** +** This is an eponymous virtual table so it does not need to be created before +** use. The optional argument to the sqlite_dbpage() table name is the +** schema for the database file that is to be read. The default schema is +** "main". +** +** The data field of sqlite_dbpage table can be updated. The new +** value must be a BLOB which is the correct page size, otherwise the +** update fails. Rows may not be deleted or inserted. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -28,6 +41,7 @@ typedef struct DbpageCursor DbpageCursor; struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ int pgno; /* Current page number */ + int mxPgno; /* Last page to visit on this scan */ }; struct DbpageTable { @@ -65,7 +79,7 @@ static int dbpageConnect( iDb = 0; } rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)"); + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; @@ -99,7 +113,26 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){ ** 1 pgno=?1 */ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + } + if( pIdxInfo->nOrderBy>=1 + && pIdxInfo->aOrderBy[0].iColumn<=0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } return SQLITE_OK; } @@ -143,8 +176,7 @@ static int dbpageNext(sqlite3_vtab_cursor *pCursor){ static int dbpageEof(sqlite3_vtab_cursor *pCursor){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; - DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; - return pCsr->pgno >= pTab->nPage; + return pCsr->pgno > pCsr->mxPgno; } static int dbpageFilter( @@ -157,13 +189,20 @@ static int dbpageFilter( int rc = SQLITE_OK; Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; - if( idxNum==1 ){ - pCsr->pgno = sqlite3_value_int(argv[0]); - }else{ - pCsr->pgno = 0; - } pTab->szPage = sqlite3BtreeGetPageSize(pBt); pTab->nPage = sqlite3BtreeLastPage(pBt); + if( idxNum==1 ){ + pCsr->pgno = sqlite3_value_int(argv[0]); + if( pCsr->pgno<1 || pCsr->pgno>pTab->nPage ){ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + }else{ + pCsr->mxPgno = pCsr->pgno; + } + }else{ + pCsr->pgno = 1; + pCsr->mxPgno = pTab->nPage; + } return rc; } @@ -205,6 +244,55 @@ static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ return SQLITE_OK; } +static int dbpageUpdate( + sqlite3_vtab *pVtab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + DbpageTable *pTab = (DbpageTable *)pVtab; + int pgno; + DbPage *pDbPage = 0; + int rc = SQLITE_OK; + char *zErr = 0; + + if( argc==1 ){ + zErr = "cannot delete"; + goto update_fail; + } + pgno = sqlite3_value_int(argv[0]); + if( pgno<1 || pgno>pTab->nPage ){ + zErr = "bad page number"; + goto update_fail; + } + if( sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + if( sqlite3_value_type(argv[3])!=SQLITE_BLOB + || sqlite3_value_bytes(argv[3])!=pTab->szPage + ){ + zErr = "bad page value"; + goto update_fail; + } + rc = sqlite3PagerGet(pTab->pPager, pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + memcpy(sqlite3PagerGetData(pDbPage), + sqlite3_value_blob(argv[3]), + pTab->szPage); + } + } + sqlite3PagerUnref(pDbPage); + return rc; + +update_fail: + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); + return SQLITE_ERROR; +} + /* ** Invoke this routine to register the "dbpage" virtual table module */ @@ -223,7 +311,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageEof, /* xEof - check for end of scan */ dbpageColumn, /* xColumn - read data */ dbpageRowid, /* xRowid - read data */ - 0, /* xUpdate */ + dbpageUpdate, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ diff --git a/test/dbpage.test b/test/dbpage.test new file mode 100644 index 0000000000..e29d4b33a9 --- /dev/null +++ b/test/dbpage.test @@ -0,0 +1,69 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_dbpage virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix dbpage + +ifcapable !vtab||!compound { + finish_test + return +} + +do_execsql_test 100 { + PRAGMA page_size=4096; + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c; + PRAGMA integrity_check; +} {wal ok} +do_execsql_test 110 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno; +} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0D00000016'} +do_execsql_test 120 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=2; +} {2 X'0500000001'} +do_execsql_test 130 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=4; +} {4 X'0D00000016'} +do_execsql_test 140 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=5; +} {} +do_execsql_test 150 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=0; +} {} + +do_execsql_test 200 { + CREATE TEMP TABLE saved_content(x); + INSERT INTO saved_content(x) SELECT data FROM sqlite_dbpage WHERE pgno=4; + UPDATE sqlite_dbpage SET data=zeroblob(4096) WHERE pgno=4; +} {} +do_catchsql_test 210 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} +do_execsql_test 220 { + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno; +} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0000000000'} +do_execsql_test 230 { + UPDATE sqlite_dbpage SET data=(SELECT x FROM saved_content) WHERE pgno=4; +} {} +do_catchsql_test 230 { + PRAGMA integrity_check; +} {0 ok} + + + + +finish_test From ca5cf1229be7c3e60c6a2daf2c01feba6d9aeb2e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 17:13:29 +0000 Subject: [PATCH 004/156] Enable sqlite_dbpage and dbstat virtual tables in the shell, by default. FossilOrigin-Name: 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db --- Makefile.in | 2 ++ Makefile.msc | 3 ++- main.mk | 2 ++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7ea97e6195..5e3f14279e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -572,6 +572,8 @@ SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 diff --git a/Makefile.msc b/Makefile.msc index d59997388d..febf496ee3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1510,7 +1510,8 @@ FUZZDATA = \ # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB !ENDIF # <> diff --git a/main.mk b/main.mk index 3f83197d4d..26a0bd586d 100644 --- a/main.mk +++ b/main.mk @@ -483,6 +483,8 @@ SHELL_OPT += -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 diff --git a/manifest b/manifest index 3e606d8851..e250415fef 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases. -D 2017-10-11T15:02:53.047 -F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9 +C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault. +D 2017-10-11T17:13:29.048 +F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d +F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf +F main.mk dc4d9dfe050e6d65671cb940974bd6b76451287db334c47332552ede28325714 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49 -R 92490bca397034b7f3fba4e951f0781d +P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f +R 26ea83977ebe8e13c118f8a4befc3e5d U drh -Z ddf133cf759c24442aeeb91134927fef +Z d68d7c86064c38dfe4b5e452d55c3294 diff --git a/manifest.uuid b/manifest.uuid index 7a388fce6d..4b74c0454f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f \ No newline at end of file +01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db \ No newline at end of file From 512e6c3c5118cba59ed3fa66a586e87100b91299 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 17:51:08 +0000 Subject: [PATCH 005/156] Convert the implementation of the ".dbstat" dot-command of the command-line shell to use the sqlite_dbpage table. FossilOrigin-Name: 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 18 +++++++++++------- src/shell.c.in | 18 +++++++++++------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index e250415fef..92b51678a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault. -D 2017-10-11T17:13:29.048 +C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable. +D 2017-10-11T17:51:08.392 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -460,8 +460,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe -F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0 +F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e +F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f -R 26ea83977ebe8e13c118f8a4befc3e5d +P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db +R 32cd47bb05391f34b12cc1216f3e5e53 U drh -Z d68d7c86064c38dfe4b5e452d55c3294 +Z ef5ad8ed0ad3bdbe43b1de17eb74fe58 diff --git a/manifest.uuid b/manifest.uuid index 4b74c0454f..12bf79e9d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db \ No newline at end of file +497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index f7be941570..2b77d482e6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4970,20 +4970,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ { "schema size:", "SELECT total(length(sql)) FROM %s" }, }; - sqlite3_file *pFile = 0; int i; char *zSchemaTab; char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; unsigned char aHdr[100]; open_db(p, 0); if( p->db==0 ) return 1; - sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile); - if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){ - return 1; - } - i = pFile->pMethods->xRead(pFile, aHdr, 100, 0); - if( i!=SQLITE_OK ){ + sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); diff --git a/src/shell.c.in b/src/shell.c.in index db4f2e1128..54a61b9456 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3610,20 +3610,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ { "schema size:", "SELECT total(length(sql)) FROM %s" }, }; - sqlite3_file *pFile = 0; int i; char *zSchemaTab; char *zDb = nArg>=2 ? azArg[1] : "main"; + sqlite3_stmt *pStmt = 0; unsigned char aHdr[100]; open_db(p, 0); if( p->db==0 ) return 1; - sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile); - if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){ - return 1; - } - i = pFile->pMethods->xRead(pFile, aHdr, 100, 0); - if( i!=SQLITE_OK ){ + sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", + -1, &pStmt, 0); + sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC); + if( sqlite3_step(pStmt)==SQLITE_ROW + && sqlite3_column_bytes(pStmt,0)>100 + ){ + memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + sqlite3_finalize(pStmt); + }else{ raw_printf(stderr, "unable to read database header\n"); + sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); From 36187fe8c2b90068f3262f962654ba5632b530e7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Oct 2017 18:00:34 +0000 Subject: [PATCH 006/156] Add new extension "checkfreelist", which uses sqlite_dbpage to check that there are no invalid entries on the database free-list. FossilOrigin-Name: 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 --- ext/misc/checkfreelist.c | 291 +++++++++++++++++++++++++++++++++++++++ manifest | 14 +- manifest.uuid | 2 +- test/checkfreelist.test | 114 +++++++++++++++ 4 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 ext/misc/checkfreelist.c create mode 100644 test/checkfreelist.test diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c new file mode 100644 index 0000000000..cec03b7999 --- /dev/null +++ b/ext/misc/checkfreelist.c @@ -0,0 +1,291 @@ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + int nData = sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && inPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} diff --git a/manifest b/manifest index 92b51678a2..d4e28e2d73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable. -D 2017-10-11T17:51:08.392 +C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list. +D 2017-10-11T18:00:34.689 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,6 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 +F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -647,6 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a +F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1658,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db -R 32cd47bb05391f34b12cc1216f3e5e53 -U drh -Z ef5ad8ed0ad3bdbe43b1de17eb74fe58 +P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 +R 7024a507e1ac7d6985bebba168f4b31f +U dan +Z 54a66c878cf2157bd3590abc0ee2a612 diff --git a/manifest.uuid b/manifest.uuid index 12bf79e9d8..e6ec495019 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 \ No newline at end of file +21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 \ No newline at end of file diff --git a/test/checkfreelist.test b/test/checkfreelist.test new file mode 100644 index 0000000000..f16d8b57bb --- /dev/null +++ b/test/checkfreelist.test @@ -0,0 +1,114 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the checkfreelist extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix checkfreelist + +ifcapable !vtab||!compound { + finish_test + return +} + +if {[file exists ../checkfreelist.so]==0} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); +} + +db enable_load_extension 1 +do_execsql_test 1.1 { + SELECT load_extension('../checkfreelist.so'); +} {{}} + +do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.3 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + DELETE FROM t1 WHERE rowid%3; + PRAGMA freelist_count; +} {6726} + +do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.5 { + WITH freelist_trunk(i, d, n) AS ( + SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 + UNION ALL + SELECT n, data, sqlite_readint32(data) + FROM freelist_trunk, sqlite_dbpage WHERE pgno=n + ) + SELECT i FROM freelist_trunk WHERE i!=1; +} { + 10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235 + 4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5 +} + +do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} + +proc set_int {blob idx newval} { + binary scan $blob I* ints + lset ints $idx $newval + binary format I* $ints +} +db func set_int set_int + +proc get_int {blob idx} { + binary scan $blob I* ints + lindex $ints $idx +} +db func get_int get_int + +do_execsql_test 1.7 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, get_int(data, 1)-1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{free-list count mismatch: actual=6725 header=6726}} + +do_execsql_test 1.8 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.9 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, 0) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.10 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, get_int(data, 1)+1, 0) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 247 of trunk page 5)}} + +finish_test + From 7d157f91f4a29e8f237f1f1d0705271a3fe45f5f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Oct 2017 18:21:44 +0000 Subject: [PATCH 007/156] Check that the leaf count on each freelist trunk page is in range as part of checkfreelist processing. FossilOrigin-Name: 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec --- ext/misc/checkfreelist.c | 8 ++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- test/checkfreelist.test | 9 +++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c index cec03b7999..345f26c223 100644 --- a/ext/misc/checkfreelist.c +++ b/ext/misc/checkfreelist.c @@ -170,6 +170,14 @@ static int checkFreelist( u32 iNext = get4byte(&aData[0]); u32 nLeaf = get4byte(&aData[4]); + if( nLeaf>((nData/4)-2-6) ){ + rc = checkFreelistError(pzOut, + "leaf count out of range (%d) on trunk page %d", + (int)nLeaf, (int)iTrunk + ); + nLeaf = (nData/4) - 2 - 6; + } + nFree += 1+nLeaf; if( iNext>nPage ){ rc = checkFreelistError(pzOut, diff --git a/manifest b/manifest index d4e28e2d73..7a6b18c230 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list. -D 2017-10-11T18:00:34.689 +C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing. +D 2017-10-11T18:21:44.429 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c +F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -648,7 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a -F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f +F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 -R 7024a507e1ac7d6985bebba168f4b31f +P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 +R f25a07255c10ac9143e97a5d9d8e737e U dan -Z 54a66c878cf2157bd3590abc0ee2a612 +Z fe2b858df1a95da0fbc85fd6ca1ee6ea diff --git a/manifest.uuid b/manifest.uuid index e6ec495019..a9df98f06f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 \ No newline at end of file +4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec \ No newline at end of file diff --git a/test/checkfreelist.test b/test/checkfreelist.test index f16d8b57bb..93e4ecc234 100644 --- a/test/checkfreelist.test +++ b/test/checkfreelist.test @@ -110,5 +110,14 @@ do_execsql_test 1.10 { ROLLBACK; } {{leaf page 0 is out of range (child 247 of trunk page 5)}} +do_execsql_test 1.11 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, 249) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf count out of range (249) on trunk page 5}} + finish_test From f294ce648b121fc5ed1c8e6456ed6b03fac34b13 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 11 Oct 2017 18:26:26 +0000 Subject: [PATCH 008/156] Add the checkfreelist extension to the command-line shell. FossilOrigin-Name: 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 --- ext/misc/checkfreelist.c | 2 +- manifest | 18 +-- manifest.uuid | 2 +- src/shell.c | 302 +++++++++++++++++++++++++++++++++++++++ src/shell.c.in | 2 + 5 files changed, 315 insertions(+), 11 deletions(-) diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c index 345f26c223..cd2801e040 100644 --- a/ext/misc/checkfreelist.c +++ b/ext/misc/checkfreelist.c @@ -223,7 +223,7 @@ static void checkfreelist_function( sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( nArg==1 ); - zDb = sqlite3_value_text(apArg[0]); + zDb = (const char*)sqlite3_value_text(apArg[0]); rc = checkFreelist(db, zDb, &zOut); if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); diff --git a/manifest b/manifest index 7a6b18c230..c2dd3cb1eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing. -D 2017-10-11T18:21:44.429 +C Add\sthe\scheckfreelist\sextension\sto\sthe\scommand-line\sshell. +D 2017-10-11T18:26:26.636 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e +F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -461,8 +461,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e -F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 +F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57 +F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 -R f25a07255c10ac9143e97a5d9d8e737e -U dan -Z fe2b858df1a95da0fbc85fd6ca1ee6ea +P 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec +R 0bf65201c138febf2eab9baca075a19d +U drh +Z f892c9c71f9811efc31d97f9328db29a diff --git a/manifest.uuid b/manifest.uuid index a9df98f06f..eb5db27581 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec \ No newline at end of file +48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2b77d482e6..6c14db9fdd 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2156,6 +2156,307 @@ int sqlite3_completion_init( } /************************* End ../ext/misc/completion.c ********************/ +/************************* Begin ../ext/misc/checkfreelist.c ******************/ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + int nData = sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + if( nLeaf>((nData/4)-2-6) ){ + rc = checkFreelistError(pzOut, + "leaf count out of range (%d) on trunk page %d", + (int)nLeaf, (int)iTrunk + ); + nLeaf = (nData/4) - 2 - 6; + } + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && inPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = (const char*)sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} + +/************************* End ../ext/misc/checkfreelist.c ********************/ #if defined(SQLITE_ENABLE_SESSION) /* @@ -4246,6 +4547,7 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } diff --git a/src/shell.c.in b/src/shell.c.in index 54a61b9456..de170887f7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -796,6 +796,7 @@ static void shellAddSchemaName( INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c +INCLUDE ../ext/misc/checkfreelist.c #if defined(SQLITE_ENABLE_SESSION) /* @@ -2886,6 +2887,7 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } From 0a386468065688eadf2d3e8fcfc22b3c3bad90ba Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Oct 2017 14:13:20 +0000 Subject: [PATCH 009/156] Fix the "snapshot-tarball" makefile target, which was broken by the shell.c change of check-in [36acc0a9]. FossilOrigin-Name: c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/mkautoconfamal.sh | 2 +- tool/warnings-clang.sh | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index f63bc613e1..2e528e87e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\strunk.\s\sAll\schanges\sare\son\smakefiles\sand\stest\sscripts.\nThere\sare\sno\score\scode\schanges. -D 2017-10-12T14:03:09.021 +C Fix\sthe\s"snapshot-tarball"\smakefile\starget,\swhich\swas\sbroken\sby\sthe\sshell.c\nchange\sof\scheck-in\s[36acc0a9]. +D 2017-10-12T14:13:20.758 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20 @@ -1585,7 +1585,7 @@ F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 -F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 +F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c736fdeca5e F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4 @@ -1632,7 +1632,7 @@ F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f -F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 +F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 8a4acea31e0f9c562949a2d767329533c0930d699ea19c6704c0ca0aa9154068 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85 @@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d0ceb8dcdef92cd72307e532a4a6c269b2c458fecb0bbede0bb941099eebc5b 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956 -R 55fc95c237c6d381b00d7c404f68ae59 +P 1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad +R 8088fa2748c2b3c07e15e47d6681488a U drh -Z 9ae511da21bc6dca261bd7a352a2c613 +Z 449b637c9672f868a961d2fed56fa177 diff --git a/manifest.uuid b/manifest.uuid index 9e90e52ce3..0ade2a082b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad \ No newline at end of file +c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339 \ No newline at end of file diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 4864ee85c3..7cd7da35f6 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -51,7 +51,7 @@ cp sqlite3.h $TMPSPACE cp sqlite3ext.h $TMPSPACE cp $TOP/sqlite3.1 $TMPSPACE cp $TOP/sqlite3.pc.in $TMPSPACE -cp $TOP/src/shell.c $TMPSPACE +cp shell.c $TMPSPACE cp $TOP/src/sqlite3.rc $TMPSPACE cp $TOP/tool/Replace.cs $TMPSPACE diff --git a/tool/warnings-clang.sh b/tool/warnings-clang.sh index 7a0aa4bce7..6dcc086d2f 100644 --- a/tool/warnings-clang.sh +++ b/tool/warnings-clang.sh @@ -3,12 +3,12 @@ # Run this script in a directory with a working makefile to check for # compiler warnings in SQLite. # -rm -f sqlite3.c -make sqlite3.c +rm -f sqlite3.c shell.c +make sqlite3.c shell.c echo '************* FTS4 and RTREE ****************' scan-build gcc -c -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ -DSQLITE_DEBUG -DSQLITE_ENABLE_STAT3 sqlite3.c 2>&1 | grep -v 'ANALYZE:' echo '********** ENABLE_STAT3. THREADSAFE=0 *******' scan-build gcc -c -I. -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \ -DSQLITE_DEBUG \ - sqlite3.c ../sqlite/src/shell.c -ldl 2>&1 | grep -v 'ANALYZE:' + sqlite3.c shell.c -ldl 2>&1 | grep -v 'ANALYZE:' From 6235ee5784bba91d58cf485176c0b5ad4177080e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Oct 2017 14:18:38 +0000 Subject: [PATCH 010/156] Fix a harmless compiler warning about an unused parameter. FossilOrigin-Name: de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2e528e87e0..b720d2abb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"snapshot-tarball"\smakefile\starget,\swhich\swas\sbroken\sby\sthe\sshell.c\nchange\sof\scheck-in\s[36acc0a9]. -D 2017-10-12T14:13:20.758 +C Fix\sa\sharmless\scompiler\swarning\sabout\san\sunused\sparameter. +D 2017-10-12T14:18:38.182 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20 @@ -445,7 +445,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 62f88892d3a2c68cff6e8f96c81c5dfe5178eace887880c36364aabe4d8d6422 +F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c @@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad -R 8088fa2748c2b3c07e15e47d6681488a +P c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339 +R 6b8c8a980bca85216c2bbb28216f8b36 U drh -Z 449b637c9672f868a961d2fed56fa177 +Z ee2d81a451eaa5a0334a9c3d00838821 diff --git a/manifest.uuid b/manifest.uuid index 0ade2a082b..0b314a09be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339 \ No newline at end of file +de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2ddca9a5f7..dbb7636ca7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1208,6 +1208,8 @@ static int jrnlBufferSize(Pager *pPager){ assert( isOpen(pPager->fd) ); dc = sqlite3OsDeviceCharacteristics(pPager->fd); +#else + UNUSED_PARAMETER(pPager); #endif #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE From 277ee81babd94ce08cda6c57596759dec457f35b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 12 Oct 2017 19:50:28 +0000 Subject: [PATCH 011/156] Create the new ext/repair folder and move checkfreelist.c there. Remove checkfreelist.c from the command-line shell (undoing check-in [48418f2e]). FossilOrigin-Name: dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab --- ext/repair/README.md | 16 ++ ext/{misc => repair}/checkfreelist.c | 0 manifest | 18 +- manifest.uuid | 2 +- src/shell.c | 302 --------------------------- src/shell.c.in | 2 - 6 files changed, 27 insertions(+), 313 deletions(-) create mode 100644 ext/repair/README.md rename ext/{misc => repair}/checkfreelist.c (100%) diff --git a/ext/repair/README.md b/ext/repair/README.md new file mode 100644 index 0000000000..927ceb7c44 --- /dev/null +++ b/ext/repair/README.md @@ -0,0 +1,16 @@ +This folder contains extensions and utility programs intended to analyze +live database files, detect problems, and possibly fix them. + +As SQLite is being used on larger and larger databases, database sizes +are growing into the terabyte range. At that size, hardware malfunctions +and/or cosmic rays will occasionally corrupt a database file. Detecting +problems and fixing errors a terabyte-sized databases can take hours or days, +and it is undesirable to take applications that depend on the databases +off-line for such a long time. +The utilities in the folder are intended to provide mechanisms for +detecting and fixing problems in large databases while those databases +are in active use. + +The utilities and extensions in this folder are experimental and under +active development at the time of this writing (2017-10-12). If and when +they stabilize, this README will be updated to reflect that fact. diff --git a/ext/misc/checkfreelist.c b/ext/repair/checkfreelist.c similarity index 100% rename from ext/misc/checkfreelist.c rename to ext/repair/checkfreelist.c diff --git a/manifest b/manifest index e5589b7672..5c8309403b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\sthe\s3.21\sbranch. -D 2017-10-12T01:24:36.582 +C Create\sthe\snew\sext/repair\sfolder\sand\smove\scheckfreelist.c\sthere.\s\sRemove\ncheckfreelist.c\sfrom\sthe\scommand-line\sshell\s(undoing\scheck-in\s[48418f2e]). +D 2017-10-12T19:50:28.969 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 523fd246fd801689c766945f6450615d3784fda2dc04993bcafc796ac6afd8df @@ -259,7 +259,6 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 -F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -326,6 +325,8 @@ F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078da04feb F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a +F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 +F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 w ext/misc/checkfreelist.c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -461,8 +462,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57 -F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5 +F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e +F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1660,7 +1661,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b -R 9b46eb5073fd5db6110bab2092230089 +P 18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857 +Q -48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 +R 4a0d06893cd053abce709ab7740cd918 U drh -Z b5c9a13e106677edbcab94b5f19597c5 +Z 46634faecfbbf65605db6fcff2dbe040 diff --git a/manifest.uuid b/manifest.uuid index f8e3904fb9..58e6b4a3f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857 \ No newline at end of file +dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 6c14db9fdd..2b77d482e6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2156,307 +2156,6 @@ int sqlite3_completion_init( } /************************* End ../ext/misc/completion.c ********************/ -/************************* Begin ../ext/misc/checkfreelist.c ******************/ -/* -** 2017 October 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module exports a single C function: -** -** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); -** -** This function checks the free-list in database zDb (one of "main", -** "temp", etc.) and reports any errors by invoking the sqlite3_log() -** function. It returns SQLITE_OK if successful, or an SQLite error -** code otherwise. It is not an error if the free-list is corrupted but -** no IO or OOM errors occur. -** -** If this file is compiled and loaded as an SQLite loadable extension, -** it adds an SQL function "checkfreelist" to the database handle, to -** be invoked as follows: -** -** SELECT checkfreelist(); -** -** This function performs the same checks as sqlite3_check_freelist(), -** except that it returns all error messages as a single text value, -** separated by newline characters. If the freelist is not corrupted -** in any way, an empty string is returned. -** -** To compile this module for use as an SQLite loadable extension: -** -** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so -*/ - -SQLITE_EXTENSION_INIT1 - -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -# define ALWAYS(X) 1 -# define NEVER(X) 0 - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -/* -** Execute a single PRAGMA statement and return the integer value returned -** via output parameter (*pnOut). -** -** The SQL statement passed as the third argument should be a printf-style -** format string containing a single "%s" which will be replace by the -** value passed as the second argument. e.g. -** -** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) -** -** executes "PRAGMA main.page_count" and stores the results in (*pnOut). -*/ -static int sqlGetInteger( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Database name ("main", "temp" etc.) */ - const char *zFmt, /* SQL statement format */ - u32 *pnOut /* OUT: Integer value */ -){ - int rc, rc2; - char *zSql; - sqlite3_stmt *pStmt = 0; - int bOk = 0; - - zSql = sqlite3_mprintf(zFmt, zDb); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnOut = (u32)sqlite3_column_int(pStmt, 0); - bOk = 1; - } - - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; - return rc; -} - -/* -** Argument zFmt must be a printf-style format string and must be -** followed by its required arguments. If argument pzOut is NULL, -** then the results of printf()ing the format string are passed to -** sqlite3_log(). Otherwise, they are appended to the string -** at (*pzOut). -*/ -static int checkFreelistError(char **pzOut, const char *zFmt, ...){ - int rc = SQLITE_OK; - char *zErr = 0; - va_list ap; - - va_start(ap, zFmt); - zErr = sqlite3_vmprintf(zFmt, ap); - if( zErr==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( pzOut ){ - *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); - if( *pzOut==0 ) rc = SQLITE_NOMEM; - }else{ - sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); - } - sqlite3_free(zErr); - } - va_end(ap); - return rc; -} - -static int checkFreelist( - sqlite3 *db, - const char *zDb, - char **pzOut -){ - /* This query returns one row for each page on the free list. Each row has - ** two columns - the page number and page content. */ - const char *zTrunk = - "WITH freelist_trunk(i, d, n) AS (" - "SELECT 1, NULL, sqlite_readint32(data, 32) " - "FROM sqlite_dbpage(:1) WHERE pgno=1 " - "UNION ALL " - "SELECT n, data, sqlite_readint32(data) " - "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " - ")" - "SELECT i, d FROM freelist_trunk WHERE i!=1;"; - - int rc, rc2; /* Return code */ - sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ - u32 nPage = 0; /* Number of pages in db */ - u32 nExpected = 0; /* Expected number of free pages */ - u32 nFree = 0; /* Number of pages on free list */ - - if( zDb==0 ) zDb = "main"; - - if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) - || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) - ){ - return rc; - } - - rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); - while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ - u32 i; - u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); - const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - int nData = sqlite3_column_bytes(pTrunk, 1); - u32 iNext = get4byte(&aData[0]); - u32 nLeaf = get4byte(&aData[4]); - - if( nLeaf>((nData/4)-2-6) ){ - rc = checkFreelistError(pzOut, - "leaf count out of range (%d) on trunk page %d", - (int)nLeaf, (int)iTrunk - ); - nLeaf = (nData/4) - 2 - 6; - } - - nFree += 1+nLeaf; - if( iNext>nPage ){ - rc = checkFreelistError(pzOut, - "trunk page %d is out of range", (int)iNext - ); - } - - for(i=0; rc==SQLITE_OK && inPage ){ - rc = checkFreelistError(pzOut, - "leaf page %d is out of range (child %d of trunk page %d)", - (int)iLeaf, (int)i, (int)iTrunk - ); - } - } - } - - if( rc==SQLITE_OK && nFree!=nExpected ){ - rc = checkFreelistError(pzOut, - "free-list count mismatch: actual=%d header=%d", - (int)nFree, (int)nExpected - ); - } - - rc2 = sqlite3_finalize(pTrunk); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ - return checkFreelist(db, zDb, 0); -} - -static void checkfreelist_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const char *zDb; - int rc; - char *zOut = 0; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( nArg==1 ); - zDb = (const char*)sqlite3_value_text(apArg[0]); - rc = checkFreelist(db, zDb, &zOut); - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - - sqlite3_free(zOut); -} - -/* -** An SQL function invoked as follows: -** -** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob -*/ -static void readint_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const u8 *zBlob; - int nBlob; - int iOff = 0; - u32 iRet = 0; - - if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error( - pCtx, "wrong number of arguments to function sqlite_readint32()", -1 - ); - return; - } - if( nArg==2 ){ - iOff = sqlite3_value_int(apArg[1]); - } - - zBlob = sqlite3_value_blob(apArg[0]); - nBlob = sqlite3_value_bytes(apArg[0]); - - if( nBlob>=(iOff+4) ){ - iRet = get4byte(&zBlob[iOff]); - } - - sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); -} - -/* -** Register the SQL functions. -*/ -static int cflRegister(sqlite3 *db){ - int rc = sqlite3_create_function( - db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_create_function( - db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 - ); - return rc; -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 - -#endif -int sqlite3_checkfreelist_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return cflRegister(db); -} - -/************************* End ../ext/misc/checkfreelist.c ********************/ #if defined(SQLITE_ENABLE_SESSION) /* @@ -4547,7 +4246,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); - sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } diff --git a/src/shell.c.in b/src/shell.c.in index de170887f7..54a61b9456 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -796,7 +796,6 @@ static void shellAddSchemaName( INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c -INCLUDE ../ext/misc/checkfreelist.c #if defined(SQLITE_ENABLE_SESSION) /* @@ -2887,7 +2886,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); - sqlite3_checkfreelist_init(p->db, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); } From c318f730a44618db5a84bba635be5b53ab876dcc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 15:06:06 +0000 Subject: [PATCH 012/156] Move some test logic out of tclsqlite.c and into auxiliary test_*.c files. This is a work in progress. FossilOrigin-Name: 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf --- Makefile.in | 2 + Makefile.msc | 2 + main.mk | 2 + manifest | 24 +- manifest.uuid | 2 +- src/tclsqlite.c | 636 +++-------------------------------------------- src/test_md5.c | 450 +++++++++++++++++++++++++++++++++ src/test_tclsh.c | 196 +++++++++++++++ 8 files changed, 697 insertions(+), 617 deletions(-) create mode 100644 src/test_md5.c create mode 100644 src/test_tclsh.c diff --git a/Makefile.in b/Makefile.in index 5295c2b032..8c306629e0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -396,6 +396,7 @@ TESTSRC = \ $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ + $(TOP)/src/test_md5.c \ $(TOP)/src/test_multiplex.c \ $(TOP)/src/test_mutex.c \ $(TOP)/src/test_onefile.c \ @@ -407,6 +408,7 @@ TESTSRC = \ $(TOP)/src/test_server.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ + $(TOP)/src/test_tclsh.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ diff --git a/Makefile.msc b/Makefile.msc index da71df89f0..e4a72e872b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1374,6 +1374,7 @@ TESTSRC = \ $(TOP)\src\test_intarray.c \ $(TOP)\src\test_journal.c \ $(TOP)\src\test_malloc.c \ + $(TOP)\src\test_md5.c \ $(TOP)\src\test_multiplex.c \ $(TOP)\src\test_mutex.c \ $(TOP)\src\test_onefile.c \ @@ -1385,6 +1386,7 @@ TESTSRC = \ $(TOP)\src\test_server.c \ $(TOP)\src\test_superlock.c \ $(TOP)\src\test_syscall.c \ + $(TOP)\src\test_tclsh.c \ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ diff --git a/main.mk b/main.mk index d03ee8d1d3..84d1a3eb76 100644 --- a/main.mk +++ b/main.mk @@ -308,6 +308,7 @@ TESTSRC = \ $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ + $(TOP)/src/test_md5.c \ $(TOP)/src/test_multiplex.c \ $(TOP)/src/test_mutex.c \ $(TOP)/src/test_onefile.c \ @@ -320,6 +321,7 @@ TESTSRC = \ $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ + $(TOP)/src/test_tclsh.c \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ diff --git a/manifest b/manifest index 3360e30a3a..250e0c8001 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\ssqlite_dbpage\svirtual\stable\s(enabled\susing\sSQLITE_ENABLE_DBPAGE_VTAB).\nMake\sthat\svirtual\stable\sand\sdbstat\savailable\sto\sthe\scommand-line\sshell. -D 2017-10-12T20:37:20.278 -F Makefile.in cb88ca5a6d8e116a50bd19bf477384498df475c5463cbbf53f36624ce308ed62 +C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress. +D 2017-10-13T15:06:06.783 +F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 918f07fee01bf858d20df6b37802161423f216da1ece14794afee12de4992e35 +F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 8be17ffd37df86bfbd696072045cd360600499876f7144313090f96ce3a37a55 +F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828 +F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -499,6 +499,7 @@ F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c 4f06a805de86be5216a127b3777ca2d5a1ff99d1a9238374ce136a47411be36c +F src/test_md5.c 7268e1e8c399d4a5e181b64ac20e1e6f3bc4dd9fc87abac02db145a3d951fa8c F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a @@ -513,6 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 +F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e @@ -1661,8 +1663,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956 dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab -R a8e3c658e72c296e51cc10d88be952b0 -T +closed dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab +P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a +R 225e4cd689e0c61d406db3daa52debdc +T *branch * tclsqlite-cleanup +T *sym-tclsqlite-cleanup * +T -sym-trunk * U drh -Z 03ff5b730193797e00df33319c2ae82d +Z 604c544fbf5b9e259c249448beb617c7 diff --git a/manifest.uuid b/manifest.uuid index 50f9e1660f..705f05a85e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a \ No newline at end of file +95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 1b9f91405e..a66fe6832e 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3546,443 +3546,16 @@ int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } #endif +/* +** If the TCLSH macro is defined to be either 1 or 2, then a main() +** routine is inserted that starts up a Tcl interpreter. When TCLSH==1 +** the interpreter works like an ordinary tclsh. When TCLSH==2 then the +** startup script is supplied by an routine named "tclsh_main_loop()" +** that must be linked separately. The TCLSH==2 technique is used to +** generate stand-alone executables based on TCL, such as +** sqlite3_analyzer.exe. +*/ #ifdef TCLSH -/***************************************************************************** -** All of the code that follows is used to build standalone TCL interpreters -** that are statically linked with SQLite. Enable these by compiling -** with -DTCLSH=n where n can be 1 or 2. An n of 1 generates a standard -** tclsh but with SQLite built in. An n of 2 generates the SQLite space -** analysis program. -*/ - -#if defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) -/* - * This code implements the MD5 message-digest algorithm. - * The algorithm is due to Ron Rivest. This code was - * written by Colin Plumb in 1993, no copyright is claimed. - * This code is in the public domain; do with it what you wish. - * - * Equivalent code is available from RSA Data Security, Inc. - * This code has been tested against that, and is equivalent, - * except that you don't need to include two pages of legalese - * with every copy. - * - * To compute the message digest of a chunk of bytes, declare an - * MD5Context structure, pass it to MD5Init, call MD5Update as - * needed on buffers full of bytes, and then call MD5Final, which - * will fill a supplied 16-byte array with the digest. - */ - -/* - * If compiled on a machine that doesn't have a 32-bit integer, - * you just set "uint32" to the appropriate datatype for an - * unsigned 32-bit integer. For example: - * - * cc -Duint32='unsigned long' md5.c - * - */ -#ifndef uint32 -# define uint32 unsigned int -#endif - -struct MD5Context { - int isInit; - uint32 buf[4]; - uint32 bits[2]; - unsigned char in[64]; -}; -typedef struct MD5Context MD5Context; - -/* - * Note: this code is harmless on little-endian machines. - */ -static void byteReverse (unsigned char *buf, unsigned longs){ - uint32 t; - do { - t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | - ((unsigned)buf[1]<<8 | buf[0]); - *(uint32 *)buf = t; - buf += 4; - } while (--longs); -} -/* The four core functions - F1 is optimized somewhat */ - -/* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) -#define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) - -/* This is the central step in the MD5 algorithm. */ -#define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) - -/* - * The core of the MD5 algorithm, this alters an existing MD5 hash to - * reflect the addition of 16 longwords of new data. MD5Update blocks - * the data and converts bytes into longwords for this routine. - */ -static void MD5Transform(uint32 buf[4], const uint32 in[16]){ - register uint32 a, b, c, d; - - a = buf[0]; - b = buf[1]; - c = buf[2]; - d = buf[3]; - - MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); - MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); - MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); - MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); - MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); - MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); - MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); - MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); - MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); - MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); - MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); - MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); - MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); - MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); - MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); - MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); - - MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); - MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); - MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); - MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); - MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); - MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); - MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); - MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); - MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); - MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); - MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); - MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); - MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); - MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); - MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); - MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); - - MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); - MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); - MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); - MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); - MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); - MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); - MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); - MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); - MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); - MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); - MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); - MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); - MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); - MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); - MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); - MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); - - MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); - MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); - MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); - MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); - MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); - MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); - MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); - MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); - MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); - MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); - MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); - MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); - MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); - MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); - MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); - MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); - - buf[0] += a; - buf[1] += b; - buf[2] += c; - buf[3] += d; -} - -/* - * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious - * initialization constants. - */ -static void MD5Init(MD5Context *ctx){ - ctx->isInit = 1; - ctx->buf[0] = 0x67452301; - ctx->buf[1] = 0xefcdab89; - ctx->buf[2] = 0x98badcfe; - ctx->buf[3] = 0x10325476; - ctx->bits[0] = 0; - ctx->bits[1] = 0; -} - -/* - * Update context to reflect the concatenation of another buffer full - * of bytes. - */ -static -void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ - uint32 t; - - /* Update bitcount */ - - t = ctx->bits[0]; - if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) - ctx->bits[1]++; /* Carry from low to high */ - ctx->bits[1] += len >> 29; - - t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ - - /* Handle any leading odd-sized chunks */ - - if ( t ) { - unsigned char *p = (unsigned char *)ctx->in + t; - - t = 64-t; - if (len < t) { - memcpy(p, buf, len); - return; - } - memcpy(p, buf, t); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - buf += t; - len -= t; - } - - /* Process data in 64-byte chunks */ - - while (len >= 64) { - memcpy(ctx->in, buf, 64); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - buf += 64; - len -= 64; - } - - /* Handle any remaining bytes of data. */ - - memcpy(ctx->in, buf, len); -} - -/* - * Final wrapup - pad to 64-byte boundary with the bit pattern - * 1 0* (64-bit count of bits processed, MSB-first) - */ -static void MD5Final(unsigned char digest[16], MD5Context *ctx){ - unsigned count; - unsigned char *p; - - /* Compute number of bytes mod 64 */ - count = (ctx->bits[0] >> 3) & 0x3F; - - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - p = ctx->in + count; - *p++ = 0x80; - - /* Bytes of padding needed to make 64 bytes */ - count = 64 - 1 - count; - - /* Pad out to 56 mod 64 */ - if (count < 8) { - /* Two lots of padding: Pad the first block to 64 bytes */ - memset(p, 0, count); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); - - /* Now fill the next block with 56 bytes */ - memset(ctx->in, 0, 56); - } else { - /* Pad block to 56 bytes */ - memset(p, 0, count-8); - } - byteReverse(ctx->in, 14); - - /* Append length in bits and transform */ - memcpy(ctx->in + 14*4, ctx->bits, 8); - - MD5Transform(ctx->buf, (uint32 *)ctx->in); - byteReverse((unsigned char *)ctx->buf, 4); - memcpy(digest, ctx->buf, 16); -} - -/* -** Convert a 128-bit MD5 digest into a 32-digit base-16 number. -*/ -static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ - static char const zEncode[] = "0123456789abcdef"; - int i, j; - - for(j=i=0; i<16; i++){ - int a = digest[i]; - zBuf[j++] = zEncode[(a>>4)&0xf]; - zBuf[j++] = zEncode[a & 0xf]; - } - zBuf[j] = 0; -} - - -/* -** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers -** each representing 16 bits of the digest and separated from each -** other by a "-" character. -*/ -static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){ - int i, j; - unsigned int x; - for(i=j=0; i<16; i+=2){ - x = digest[i]*256 + digest[i+1]; - if( i>0 ) zDigest[j++] = '-'; - sqlite3_snprintf(50-j, &zDigest[j], "%05u", x); - j += 5; - } - zDigest[j] = 0; -} - -/* -** A TCL command for md5. The argument is the text to be hashed. The -** Result is the hash in base64. -*/ -static int SQLITE_TCLAPI md5_cmd( - void*cd, - Tcl_Interp *interp, - int argc, - const char **argv -){ - MD5Context ctx; - unsigned char digest[16]; - char zBuf[50]; - void (*converter)(unsigned char*, char*); - - if( argc!=2 ){ - Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], - " TEXT\"", (char*)0); - return TCL_ERROR; - } - MD5Init(&ctx); - MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1])); - MD5Final(digest, &ctx); - converter = (void(*)(unsigned char*,char*))cd; - converter(digest, zBuf); - Tcl_AppendResult(interp, zBuf, (char*)0); - return TCL_OK; -} - -/* -** A TCL command to take the md5 hash of a file. The argument is the -** name of the file. -*/ -static int SQLITE_TCLAPI md5file_cmd( - void*cd, - Tcl_Interp *interp, - int argc, - const char **argv -){ - FILE *in; - int ofst; - int amt; - MD5Context ctx; - void (*converter)(unsigned char*, char*); - unsigned char digest[16]; - char zBuf[10240]; - - if( argc!=2 && argc!=4 ){ - Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], - " FILENAME [OFFSET AMT]\"", (char*)0); - return TCL_ERROR; - } - if( argc==4 ){ - ofst = atoi(argv[2]); - amt = atoi(argv[3]); - }else{ - ofst = 0; - amt = 2147483647; - } - in = fopen(argv[1],"rb"); - if( in==0 ){ - Tcl_AppendResult(interp,"unable to open file \"", argv[1], - "\" for reading", (char*)0); - return TCL_ERROR; - } - fseek(in, ofst, SEEK_SET); - MD5Init(&ctx); - while( amt>0 ){ - int n; - n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in); - if( n<=0 ) break; - MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); - amt -= n; - } - fclose(in); - MD5Final(digest, &ctx); - converter = (void(*)(unsigned char*,char*))cd; - converter(digest, zBuf); - Tcl_AppendResult(interp, zBuf, (char*)0); - return TCL_OK; -} - -/* -** Register the four new TCL commands for generating MD5 checksums -** with the TCL interpreter. -*/ -int Md5_Init(Tcl_Interp *interp){ - Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd, - MD5DigestToBase16, 0); - Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd, - MD5DigestToBase10x8, 0); - Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd, - MD5DigestToBase16, 0); - Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd, - MD5DigestToBase10x8, 0); - return TCL_OK; -} -#endif /* defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) */ - -#if defined(SQLITE_TEST) -/* -** During testing, the special md5sum() aggregate function is available. -** inside SQLite. The following routines implement that function. -*/ -static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ - MD5Context *p; - int i; - if( argc<1 ) return; - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( p==0 ) return; - if( !p->isInit ){ - MD5Init(p); - } - for(i=0; i -#endif - #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; @@ -4271,17 +3690,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ } #endif - /* Since the primary use case for this binary is testing of SQLite, - ** be sure to generate core files if we crash */ -#if defined(SQLITE_TEST) && defined(unix) - { struct rlimit x; - getrlimit(RLIMIT_CORE, &x); - x.rlim_cur = x.rlim_max; - setrlimit(RLIMIT_CORE, &x); - } -#endif /* SQLITE_TEST && unix */ - - /* Call sqlite3_shutdown() once before doing anything else. This is to ** test that sqlite3_shutdown() can be safely called by a process before ** sqlite3_initialize() is. */ @@ -4295,7 +3703,23 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); #endif - init_all(interp); + /* Add extensions */ +#if !defined(SQLITE_TEST) + /* Normally we only initialize the TCL extension */ + Sqlite3_Init(interp); +#else + /* For testing, do lots of extra initialization */ + { + extern void sqlite3InitTclTestLogic(Tcl_Interp*); + sqlite3InitTclTestLogic(interp); + Tcl_CreateObjCommand( + interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 + ); + } +#endif /* SQLITE_TEST */ if( argc>=2 ){ int i; char zArgc[32]; diff --git a/src/test_md5.c b/src/test_md5.c new file mode 100644 index 0000000000..b670026861 --- /dev/null +++ b/src/test_md5.c @@ -0,0 +1,450 @@ +/* +** 2017-10-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to implement an MD5 extension to TCL. +*/ +#include "sqlite3.h" +#include +#include +#include "sqlite3.h" +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +/* + * If compiled on a machine that doesn't have a 32-bit integer, + * you just set "uint32" to the appropriate datatype for an + * unsigned 32-bit integer. For example: + * + * cc -Duint32='unsigned long' md5.c + * + */ +#ifndef uint32 +# define uint32 unsigned int +#endif + +struct MD5Context { + int isInit; + uint32 buf[4]; + uint32 bits[2]; + unsigned char in[64]; +}; +typedef struct MD5Context MD5Context; + +/* + * Note: this code is harmless on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32 t; + do { + t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32 *)buf = t; + buf += 4; + } while (--longs); +} +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32 buf[4], const uint32 in[16]){ + register uint32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +static void MD5Init(MD5Context *ctx){ + ctx->isInit = 1; + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +static +void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ + uint32 t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +static void MD5Final(unsigned char digest[16], MD5Context *ctx){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + memcpy(ctx->in + 14*4, ctx->bits, 8); + + MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); +} + +/* +** Convert a 128-bit MD5 digest into a 32-digit base-16 number. +*/ +static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + + +/* +** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers +** each representing 16 bits of the digest and separated from each +** other by a "-" character. +*/ +static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){ + int i, j; + unsigned int x; + for(i=j=0; i<16; i+=2){ + x = digest[i]*256 + digest[i+1]; + if( i>0 ) zDigest[j++] = '-'; + sqlite3_snprintf(50-j, &zDigest[j], "%05u", x); + j += 5; + } + zDigest[j] = 0; +} + +/* +** A TCL command for md5. The argument is the text to be hashed. The +** Result is the hash in base64. +*/ +static int SQLITE_TCLAPI md5_cmd( + void*cd, + Tcl_Interp *interp, + int argc, + const char **argv +){ + MD5Context ctx; + unsigned char digest[16]; + char zBuf[50]; + void (*converter)(unsigned char*, char*); + + if( argc!=2 ){ + Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], + " TEXT\"", (char*)0); + return TCL_ERROR; + } + MD5Init(&ctx); + MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1])); + MD5Final(digest, &ctx); + converter = (void(*)(unsigned char*,char*))cd; + converter(digest, zBuf); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; +} + +/* +** A TCL command to take the md5 hash of a file. The argument is the +** name of the file. +*/ +static int SQLITE_TCLAPI md5file_cmd( + void*cd, + Tcl_Interp *interp, + int argc, + const char **argv +){ + FILE *in; + int ofst; + int amt; + MD5Context ctx; + void (*converter)(unsigned char*, char*); + unsigned char digest[16]; + char zBuf[10240]; + + if( argc!=2 && argc!=4 ){ + Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], + " FILENAME [OFFSET AMT]\"", (char*)0); + return TCL_ERROR; + } + if( argc==4 ){ + ofst = atoi(argv[2]); + amt = atoi(argv[3]); + }else{ + ofst = 0; + amt = 2147483647; + } + in = fopen(argv[1],"rb"); + if( in==0 ){ + Tcl_AppendResult(interp,"unable to open file \"", argv[1], + "\" for reading", (char*)0); + return TCL_ERROR; + } + fseek(in, ofst, SEEK_SET); + MD5Init(&ctx); + while( amt>0 ){ + int n; + n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in); + if( n<=0 ) break; + MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + amt -= n; + } + fclose(in); + MD5Final(digest, &ctx); + converter = (void(*)(unsigned char*,char*))cd; + converter(digest, zBuf); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; +} + +/* +** Register the four new TCL commands for generating MD5 checksums +** with the TCL interpreter. +*/ +int Md5_Init(Tcl_Interp *interp){ + Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd, + MD5DigestToBase16, 0); + Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd, + MD5DigestToBase10x8, 0); + Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd, + MD5DigestToBase16, 0); + Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd, + MD5DigestToBase10x8, 0); + return TCL_OK; +} + +/* +** During testing, the special md5sum() aggregate function is available. +** inside SQLite. The following routines implement that function. +*/ +static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ + MD5Context *p; + int i; + if( argc<1 ) return; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + MD5Init(p); + } + for(i=0; i +#endif + +/* Forward declaration */ +static int SQLITE_TCLAPI load_testfixture_extensions( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +); + +/* +** This routine is the primary export of this file. +** +** Configure the interpreter passed as the first argument to have access +** to the commands and linked variables that make up: +** +** * the [sqlite3] extension itself, +** +** * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and +** +** * If SQLITE_TEST is set, the various test interfaces used by the Tcl +** test suite. +*/ +void sqlite3InitTclTestLogic(Tcl_Interp *interp){ + /* Since the primary use case for this binary is testing of SQLite, + ** be sure to generate core files if we crash */ +#if defined(unix) + { struct rlimit x; + getrlimit(RLIMIT_CORE, &x); + x.rlim_cur = x.rlim_max; + setrlimit(RLIMIT_CORE, &x); + } +#endif /* unix */ + + { + extern int Sqlite3_Init(Tcl_Interp*); + extern int Sqliteconfig_Init(Tcl_Interp*); + extern int Sqlitetest1_Init(Tcl_Interp*); + extern int Sqlitetest2_Init(Tcl_Interp*); + extern int Sqlitetest3_Init(Tcl_Interp*); + extern int Sqlitetest4_Init(Tcl_Interp*); + extern int Sqlitetest5_Init(Tcl_Interp*); + extern int Sqlitetest6_Init(Tcl_Interp*); + extern int Sqlitetest7_Init(Tcl_Interp*); + extern int Sqlitetest8_Init(Tcl_Interp*); + extern int Sqlitetest9_Init(Tcl_Interp*); + extern int Sqlitetestasync_Init(Tcl_Interp*); + extern int Sqlitetest_autoext_Init(Tcl_Interp*); + extern int Sqlitetest_blob_Init(Tcl_Interp*); + extern int Sqlitetest_demovfs_Init(Tcl_Interp *); + extern int Sqlitetest_func_Init(Tcl_Interp*); + extern int Sqlitetest_hexio_Init(Tcl_Interp*); + extern int Sqlitetest_init_Init(Tcl_Interp*); + extern int Sqlitetest_malloc_Init(Tcl_Interp*); + extern int Sqlitetest_mutex_Init(Tcl_Interp*); + extern int Sqlitetestschema_Init(Tcl_Interp*); + extern int Sqlitetestsse_Init(Tcl_Interp*); + extern int Sqlitetesttclvar_Init(Tcl_Interp*); + extern int Sqlitetestfs_Init(Tcl_Interp*); + extern int SqlitetestThread_Init(Tcl_Interp*); + extern int SqlitetestOnefile_Init(); + extern int SqlitetestOsinst_Init(Tcl_Interp*); + extern int Sqlitetestbackup_Init(Tcl_Interp*); + extern int Sqlitetestintarray_Init(Tcl_Interp*); + extern int Sqlitetestvfs_Init(Tcl_Interp *); + extern int Sqlitetestrtree_Init(Tcl_Interp*); + extern int Sqlitequota_Init(Tcl_Interp*); + extern int Sqlitemultiplex_Init(Tcl_Interp*); + extern int SqliteSuperlock_Init(Tcl_Interp*); + extern int SqlitetestSyscall_Init(Tcl_Interp*); +#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + extern int TestSession_Init(Tcl_Interp*); +#endif + extern int Md5_Init(Tcl_Interp*); + extern int Fts5tcl_Init(Tcl_Interp *); + extern int SqliteRbu_Init(Tcl_Interp*); + extern int Sqlitetesttcl_Init(Tcl_Interp*); +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif +#ifdef SQLITE_ENABLE_ZIPVFS + extern int Zipvfs_Init(Tcl_Interp*); +#endif + + Sqlite3_Init(interp); +#ifdef SQLITE_ENABLE_ZIPVFS + Zipvfs_Init(interp); +#endif + Md5_Init(interp); + Sqliteconfig_Init(interp); + Sqlitetest1_Init(interp); + Sqlitetest2_Init(interp); + Sqlitetest3_Init(interp); + Sqlitetest4_Init(interp); + Sqlitetest5_Init(interp); + Sqlitetest6_Init(interp); + Sqlitetest7_Init(interp); + Sqlitetest8_Init(interp); + Sqlitetest9_Init(interp); + Sqlitetestasync_Init(interp); + Sqlitetest_autoext_Init(interp); + Sqlitetest_blob_Init(interp); + Sqlitetest_demovfs_Init(interp); + Sqlitetest_func_Init(interp); + Sqlitetest_hexio_Init(interp); + Sqlitetest_init_Init(interp); + Sqlitetest_malloc_Init(interp); + Sqlitetest_mutex_Init(interp); + Sqlitetestschema_Init(interp); + Sqlitetesttclvar_Init(interp); + Sqlitetestfs_Init(interp); + SqlitetestThread_Init(interp); + SqlitetestOnefile_Init(); + SqlitetestOsinst_Init(interp); + Sqlitetestbackup_Init(interp); + Sqlitetestintarray_Init(interp); + Sqlitetestvfs_Init(interp); + Sqlitetestrtree_Init(interp); + Sqlitequota_Init(interp); + Sqlitemultiplex_Init(interp); + SqliteSuperlock_Init(interp); + SqlitetestSyscall_Init(interp); +#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + TestSession_Init(interp); +#endif + Fts5tcl_Init(interp); + SqliteRbu_Init(interp); + Sqlitetesttcl_Init(interp); + +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + Sqlitetestfts3_Init(interp); +#endif + + Tcl_CreateObjCommand( + interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 + ); + } +} + +/* tclcmd: load_testfixture_extensions +*/ +static int SQLITE_TCLAPI load_testfixture_extensions( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + + Tcl_Interp *slave; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SLAVE"); + return TCL_ERROR; + } + + slave = Tcl_GetSlave(interp, Tcl_GetString(objv[1])); + if( !slave ){ + return TCL_ERROR; + } + + sqlite3InitTclTestLogic(slave); + return TCL_OK; +} From 1df6470d359e9e9cc4eacbd1932dce5354b8b18a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 15:56:26 +0000 Subject: [PATCH 013/156] When compiling with SQLITE_TEST, extra options are available on the "DB version" command for controlling the internal state of the SqliteDb object, for testing purposes. FossilOrigin-Name: bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 --- Makefile.in | 1 + Makefile.msc | 1 + main.mk | 1 + manifest | 23 ++++---- manifest.uuid | 2 +- src/tclsqlite.c | 126 +++++++++++++------------------------------ test/scanstatus.test | 4 +- 7 files changed, 52 insertions(+), 106 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8c306629e0..d474222fb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1119,6 +1119,7 @@ TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c diff --git a/Makefile.msc b/Makefile.msc index e4a72e872b..0611d8a8b3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2111,6 +2111,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) diff --git a/main.mk b/main.mk index 84d1a3eb76..bdbb56e229 100644 --- a/main.mk +++ b/main.mk @@ -803,6 +803,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ diff --git a/manifest b/manifest index 250e0c8001..74a206741c 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress. -D 2017-10-13T15:06:06.783 -F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b +C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes. +D 2017-10-13T15:56:26.514 +F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558 +F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61 +F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2 +F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -1157,7 +1157,7 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 +F test/scanstatus.test 1ba0e2ee25dcd1d55ec770803b19832cffaecbf0b15d376807759ebeed3669b0 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1663,10 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a -R 225e4cd689e0c61d406db3daa52debdc -T *branch * tclsqlite-cleanup -T *sym-tclsqlite-cleanup * -T -sym-trunk * +P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf +R beaa5ad001aa19fb6dcaa6d5d2766f78 U drh -Z 604c544fbf5b9e259c249448beb617c7 +Z 978089b1abbc5f3c189d1cdd601af59d diff --git a/manifest.uuid b/manifest.uuid index 705f05a85e..ddda81051f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf \ No newline at end of file +bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index a66fe6832e..0009eab695 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3286,7 +3286,42 @@ static int SQLITE_TCLAPI DbObjCmd( ** Return the version string for this database. */ case DB_VERSION: { - Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC); + int i; + for(i=2; ibLegacyPrepare) ){ + return TCL_ERROR; + } + }else + + /* $db version -last-stmt-ptr + ** + ** Return a string which is a hex encoding of the pointer to the + ** most recent sqlite3_stmt in the statement cache. + */ + if( strcmp(zArg, "-last-stmt-ptr")==0 ){ + char zBuf[100]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%p", + pDb->stmtList ? pDb->stmtList->pStmt: 0); + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + }else +#endif /* SQLITE_TEST */ + { + Tcl_AppendResult(interp, "unknown argument: ", zArg, (char*)0); + return TCL_ERROR; + } + } + if( i==2 ){ + Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC); + } break; } @@ -3594,89 +3629,6 @@ static const char *tclsh_main_loop(void){ static const char *tclsh_main_loop(void); #endif -/* The following to TCL commands used for testing must appear in this -** file (they cannot be factored out into one of the test_*.c files where -** they belong) because they require access to the SqliteDb object. -*/ -#ifdef SQLITE_TEST -/* -** Tclcmd: db_use_legacy_prepare DB BOOLEAN -** -** The first argument to this command must be a database command created by -** [sqlite3]. If the second argument is true, then the handle is configured -** to use the sqlite3_prepare_v2() function to prepare statements. If it -** is false, sqlite3_prepare(). -*/ -static int SQLITE_TCLAPI db_use_legacy_prepare_cmd( - ClientData cd, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - Tcl_CmdInfo cmdInfo; - SqliteDb *pDb; - int bPrepare; - - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN"); - return TCL_ERROR; - } - - if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ - Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); - return TCL_ERROR; - } - pDb = (SqliteDb*)cmdInfo.objClientData; - if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){ - return TCL_ERROR; - } - - pDb->bLegacyPrepare = bPrepare; - - Tcl_ResetResult(interp); - return TCL_OK; -} - -/* -** Tclcmd: db_last_stmt_ptr DB -** -** If the statement cache associated with database DB is not empty, -** return the text representation of the most recently used statement -** handle. -*/ -static int SQLITE_TCLAPI db_last_stmt_ptr( - ClientData cd, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); - Tcl_CmdInfo cmdInfo; - SqliteDb *pDb; - sqlite3_stmt *pStmt = 0; - char zBuf[100]; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - - if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ - Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); - return TCL_ERROR; - } - pDb = (SqliteDb*)cmdInfo.objClientData; - - if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt; - if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){ - return TCL_ERROR; - } - Tcl_SetResult(interp, zBuf, TCL_VOLATILE); - - return TCL_OK; -} -#endif /* SQLITE_TEST */ - #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; @@ -3712,12 +3664,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ { extern void sqlite3InitTclTestLogic(Tcl_Interp*); sqlite3InitTclTestLogic(interp); - Tcl_CreateObjCommand( - interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 - ); - Tcl_CreateObjCommand( - interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 - ); } #endif /* SQLITE_TEST */ if( argc>=2 ){ diff --git a/test/scanstatus.test b/test/scanstatus.test index ed24d97437..fdd6476ae9 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -30,7 +30,7 @@ do_execsql_test 1.0 { } proc do_scanstatus_test {tn res} { - set stmt [db_last_stmt_ptr db] + set stmt [db version -last-stmt-ptr] set idx 0 set ret [list] while {1} { @@ -79,7 +79,7 @@ do_scanstatus_test 1.9 { } do_test 1.9 { - sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db] + sqlite3_stmt_scanstatus_reset [db version -last-stmt-ptr] } {} do_scanstatus_test 1.10 { From 8544d43e4ae349ab04aea9fe6c075627abaafeeb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 16:01:36 +0000 Subject: [PATCH 014/156] The db_use_legacy_prepare command no longer exists. Use the undocumented -use-legacy-prepare option to DB version instead. FossilOrigin-Name: ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 74a206741c..cc4d9453b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes. -D 2017-10-13T15:56:26.514 +C The\sdb_use_legacy_prepare\scommand\sno\slonger\sexists.\s\sUse\sthe\sundocumented\n-use-legacy-prepare\soption\sto\sDB\sversion\sinstead. +D 2017-10-13T16:01:36.071 F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 @@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531 +F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1663,7 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf -R beaa5ad001aa19fb6dcaa6d5d2766f78 +P bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 +R ef5c1b585866b0d225763ce3d0edba7c U drh -Z 978089b1abbc5f3c189d1cdd601af59d +Z e1aad8bb9151be888a26c10aa66d2e50 diff --git a/manifest.uuid b/manifest.uuid index ddda81051f..5b87ac07a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896 \ No newline at end of file +ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 5afc51cb7d..1e04265668 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1065,7 +1065,7 @@ test_suite "no_optimization" -description { test_suite "prepare" -description { Run tests with the db connection using sqlite3_prepare() instead of _v2(). } -dbconfig { - db_use_legacy_prepare $::dbhandle 1 + $::dbhandle version -use-legacy-prepare 1 #$::dbhandle cache size 0 } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ From 903b23022d6d39fbc18d619dfdead39277bed72d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 18:58:55 +0000 Subject: [PATCH 015/156] Add the tcl/mkccode.tcl script used to construct a single C-language source fiel for programs that combine C-code, SQLite, and TCL. Use this script to construct the sqlite3_analyzer program. FossilOrigin-Name: 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 --- Makefile.in | 10 +- Makefile.msc | 10 +- main.mk | 10 +- manifest | 20 +- manifest.uuid | 2 +- tool/mkccode.tcl | 93 ++++ tool/sqlite3_analyzer.c.in | 903 +++++++++++++++++++++++++++++++++++++ tool/tostr.tcl | 12 - 8 files changed, 1013 insertions(+), 47 deletions(-) create mode 100755 tool/mkccode.tcl create mode 100644 tool/sqlite3_analyzer.c.in delete mode 100644 tool/tostr.tcl diff --git a/Makefile.in b/Makefile.in index d474222fb8..e2934d80b1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1180,14 +1180,8 @@ valgrindtest: $(TESTPROGS) valgrindfuzz smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl - echo "#define TCLSH 2" > $@ - echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ - cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ - echo "static const char *tclsh_main_loop(void){" >> $@ - echo "static const char *zMainloop = " >> $@ - $(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ - echo "; return zMainloop; }" >> $@ +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in + $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 0611d8a8b3..bfd368c558 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2191,14 +2191,8 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP) - echo #define TCLSH 2 > $@ - echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ - copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ - echo static const char *tclsh_main_loop(void){ >> $@ - echo static const char *zMainloop = >> $@ - $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ - echo ; return zMainloop; } >> $@ +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ diff --git a/main.mk b/main.mk index bdbb56e229..18d7cd3459 100644 --- a/main.mk +++ b/main.mk @@ -780,14 +780,8 @@ tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a $(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl - echo "#define TCLSH 2" > $@ - echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ - cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ - echo "static const char *tclsh_main_loop(void){" >> $@ - echo "static const char *zMainloop = " >> $@ - tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ - echo "; return zMainloop; }" >> $@ +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl + tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) diff --git a/manifest b/manifest index c45a7ce09e..669c1b284a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Move\sa\sbunch\sof\sunrelated\stest\scode\sout\sof\stclsqlite.c\sand\sinto\ntest_*.c\sfiles.\s\sThere\sis\sstill\ssome\stest\scode\sin\stclsqlite.c,\sbut\sthe\samount\nis\sgreatly\sreduced. -D 2017-10-13T16:19:48.503 -F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294 +C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram. +D 2017-10-13T18:58:55.897 +F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5 +F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf +F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1593,6 +1593,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 +F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4 @@ -1631,11 +1632,11 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb +F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 -F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f @@ -1663,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b -R ef5c1b585866b0d225763ce3d0edba7c -T +closed ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b +P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d +R 78f34b33921be1c7a7c8587f43b33b81 U drh -Z 0c47718eb3eab9783f8c2bca1833cb14 +Z 180ea09ee34fefc9e97e7461c13719ef diff --git a/manifest.uuid b/manifest.uuid index 05172a0f2b..fabaeec3c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d \ No newline at end of file +298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 \ No newline at end of file diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl new file mode 100755 index 0000000000..41b09f1e81 --- /dev/null +++ b/tool/mkccode.tcl @@ -0,0 +1,93 @@ +#!/usr/bin/tclsh +# +# Use this script to build C-language source code for a program that uses +# tclsqlite.c together with custom TCL scripts and/or C extensions for +# either SQLite or TCL. +# +# Usage example: +# +# tclsh mktclsqliteprog.tcl demoapp.c.in >demoapp.c +# +# The demoapp.c.in file contains a mixture of C code, TCL script, and +# processing directives used by mktclsqliteprog.tcl to build the final C-code +# output file. Most lines of demoapp.c.in are copied straight through into +# the output. The following control directives are recognized: +# +# BEGIN_STRING +# +# This marks the beginning of large string literal - usually a TCL +# script of some kind. Subsequent lines of text through the first +# line that begins with END_STRING are converted into a C-language +# string literal. +# +# INCLUDE path +# +# The path argument is the name of a file to be inserted in place of +# the INCLUDE line. The path can begin with $ROOT to signify the +# root of the SQLite source tree, or $HOME to signify the directory +# that contains the demoapp.c.in input script itself. If the path does +# not begin with either $ROOT or $HOME, then it is interpreted relative +# to the current working directory. +# +# If the INCLUDE occurs in the middle of BEGIN_STRING...END_STRING +# then all of the text in the input file is converted into C-language +# string literals. +# +# None of the control directives described above will nest. Only the +# top-level input file ("demoapp.c.in" in the example) is interpreted. +# referenced files are copied verbatim. +# +if {[llength $argv]!=1} { + puts stderr "Usage: $argv0 TEMPLATE >OUTPUT" + exit 1 +} +set infile [lindex $argv 0] +set ROOT [file normalize [file dir $argv0]/..] +set HOME [file normalize [file dir $infile]] +set in [open $infile rb] +puts [subst {/* DO NOT EDIT +** +** This file was generated by \"$argv0 $infile\". +** To make changes, edit $infile then rerun the generator +** command. +*/}] +set instr 0 +while {1} { + set line [gets $in] + if {[eof $in]} break + if {[regexp {^INCLUDE (.*)} $line all path]} { + regsub {^\$ROOT\y} $path $ROOT path + regsub {^\$HOME\y} $path $HOME path + set in2 [open $path rb] + puts "/* INCLUDE $path */" + if {$instr} { + while {1} { + set line [gets $in2] + if {[eof $in2]} break + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } + } else { + puts [read $in2] + } + puts "/* END $path */" + close $in2 + continue + } + if {[regexp {^BEGIN_STRING} $line]} { + set instr 1 + puts "/* BEGIN_STRING */" + continue + } + if {[regexp {^END_STRING} $line]} { + set instr 0 + puts "/* END_STRING */" + continue + } + if {$instr} { + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" + } else { + puts $line + } +} diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in new file mode 100644 index 0000000000..54f323a427 --- /dev/null +++ b/tool/sqlite3_analyzer.c.in @@ -0,0 +1,903 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH 2 +#define SQLITE_ENABLE_DBSTAT_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +INCLUDE sqlite3.c +INCLUDE $ROOT/src/tclsqlite.c + +static const char *tclsh_main_loop(void){ +return +BEGIN_STRING +# Run this TCL script using "testfixture" in order get a report that shows +# how much disk space is used by a particular data to actually store data +# versus how much space is unused. +# + +if {[catch { + +# Argument $tname is the name of a table within the database opened by +# database handle [db]. Return true if it is a WITHOUT ROWID table, or +# false otherwise. +# +proc is_without_rowid {tname} { + set t [string map {' ''} $tname] + db eval "PRAGMA index_list = '$t'" o { + if {$o(origin) == "pk"} { + set n $o(name) + if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} { + return 1 + } + } + } + return 0 +} + +# Read and run TCL commands from standard input. Used to implement +# the --tclsh option. +# +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + + +# Get the name of the database to analyze +# +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" + puts stderr { +Analyze the SQLite3 database file specified by the "database-filename" +argument and output a report detailing size and storage efficiency +information for the database and its constituent tables and indexes. + +Options: + + --pageinfo Show how each page of the database-file is used + + --stats Output SQL text that creates a new database containing + statistics about the database that was analyzed + + --tclsh Run the built-in TCL interpreter interactively (for debugging) + + --version Show the version number of SQLite +} + exit 1 +} +set file_to_analyze {} +set flags(-pageinfo) 0 +set flags(-stats) 0 +set flags(-debug) 0 +append argv {} +foreach arg $argv { + if {[regexp {^-+pageinfo$} $arg]} { + set flags(-pageinfo) 1 + } elseif {[regexp {^-+stats$} $arg]} { + set flags(-stats) 1 + } elseif {[regexp {^-+debug$} $arg]} { + set flags(-debug) 1 + } elseif {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } elseif {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } elseif {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } elseif {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} +set true_file_size [file size $root_filename] +if {$true_file_size<512} { + puts stderr "Empty or malformed database: $root_filename" + exit 1 +} + +# Compute the total file size assuming test_multiplexor is being used. +# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled +# +set extension [file extension $root_filename] +set pattern $root_filename +append pattern {[0-3][0-9][0-9]} +foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + set extension {} +} +if {[string length $extension]>=2 && [string length $extension]<=4} { + set pattern [file rootname $root_filename] + append pattern {.[0-3][0-9][0-9]} + foreach f [glob -nocomplain $pattern] { + incr true_file_size [file size $f] + } +} + +# Open the database +# +if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { + puts stderr "error trying to open $file_to_analyze: $msg" + exit 1 +} +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + db trace ::dbtrace +} + +db eval {SELECT count(*) FROM sqlite_master} +set pageSize [expr {wide([db one {PRAGMA page_size}])}] + +if {$flags(-pageinfo)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} { + puts "$pageno $name $path" + } + exit 0 +} +if {$flags(-stats)} { + db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} + puts "BEGIN;" + puts "CREATE TABLE stats(" + puts " name STRING, /* Name of table or index */" + puts " path INTEGER, /* Path to page from root */" + puts " pageno INTEGER, /* Page number */" + puts " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" + puts " ncell INTEGER, /* Cells on page (0 for overflow) */" + puts " payload INTEGER, /* Bytes of payload on this page */" + puts " unused INTEGER, /* Bytes of unused space on this page */" + puts " mx_payload INTEGER, /* Largest payload size of all cells */" + puts " pgoffset INTEGER, /* Offset of page in file */" + puts " pgsize INTEGER /* Size of the page */" + puts ");" + db eval {SELECT quote(name) || ',' || + quote(path) || ',' || + quote(pageno) || ',' || + quote(pagetype) || ',' || + quote(ncell) || ',' || + quote(payload) || ',' || + quote(unused) || ',' || + quote(mx_payload) || ',' || + quote(pgoffset) || ',' || + quote(pgsize) AS x FROM stat} { + puts "INSERT INTO stats VALUES($x);" + } + puts "COMMIT;" + exit 0 +} + + +# In-memory database for collecting statistics. This script loops through +# the tables and indices in the database being analyzed, adding a row for each +# to an in-memory database (for which the schema is shown below). It then +# queries the in-memory db to produce the space-analysis report. +# +sqlite3 mem :memory: +if {$flags(-debug)} { + proc dbtrace {txt} {puts $txt; flush stdout;} + mem trace ::dbtrace +} +set tabledef {CREATE TABLE space_used( + name clob, -- Name of a table or index in the database file + tblname clob, -- Name of associated table + is_index boolean, -- TRUE if it is an index, false for a table + is_without_rowid boolean, -- TRUE if WITHOUT ROWID table + nentry int, -- Number of entries in the BTree + leaf_entries int, -- Number of leaf entries + depth int, -- Depth of the b-tree + payload int, -- Total amount of data stored in this table or index + ovfl_payload int, -- Total amount of data stored on overflow pages + ovfl_cnt int, -- Number of entries that use overflow + mx_payload int, -- Maximum payload size + int_pages int, -- Number of interior pages used + leaf_pages int, -- Number of leaf pages used + ovfl_pages int, -- Number of overflow pages used + int_unused int, -- Number of unused bytes on interior pages + leaf_unused int, -- Number of unused bytes on primary pages + ovfl_unused int, -- Number of unused bytes on overflow pages + gap_cnt int, -- Number of gaps in the page layout + compressed_size int -- Total bytes stored on disk +);} +mem eval $tabledef + +# Create a temporary "dbstat" virtual table. +# +db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} +db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat + ORDER BY name, path} +db eval {DROP TABLE temp.stat} + +set isCompressed 0 +set compressOverhead 0 +set depth 0 +set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } +foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { + + set is_index [expr {$name!=$tblname}] + set is_without_rowid [is_without_rowid $name] + db eval { + SELECT + sum(ncell) AS nentry, + sum((pagetype=='leaf')*ncell) AS leaf_entries, + sum(payload) AS payload, + sum((pagetype=='overflow') * payload) AS ovfl_payload, + sum(path LIKE '%+000000') AS ovfl_cnt, + max(mx_payload) AS mx_payload, + sum(pagetype=='internal') AS int_pages, + sum(pagetype=='leaf') AS leaf_pages, + sum(pagetype=='overflow') AS ovfl_pages, + sum((pagetype=='internal') * unused) AS int_unused, + sum((pagetype=='leaf') * unused) AS leaf_unused, + sum((pagetype=='overflow') * unused) AS ovfl_unused, + sum(pgsize) AS compressed_size, + max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) + AS depth + FROM temp.dbstat WHERE name = $name + } break + + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set storage [expr {$total_pages*$pageSize}] + if {!$isCompressed && $storage>$compressed_size} { + set isCompressed 1 + set compressOverhead 14 + } + + # Column 'gap_cnt' is set to the number of non-contiguous entries in the + # list of pages visited if the b-tree structure is traversed in a top-down + # fashion (each node visited before its child-tree is passed). Any overflow + # chains present are traversed from start to finish before any child-tree + # is. + # + set gap_cnt 0 + set prev 0 + db eval { + SELECT pageno, pagetype FROM temp.dbstat + WHERE name=$name + ORDER BY pageno + } { + if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} { + incr gap_cnt + } + set prev $pageno + } + mem eval { + INSERT INTO space_used VALUES( + $name, + $tblname, + $is_index, + $is_without_rowid, + $nentry, + $leaf_entries, + $depth, + $payload, + $ovfl_payload, + $ovfl_cnt, + $mx_payload, + $int_pages, + $leaf_pages, + $ovfl_pages, + $int_unused, + $leaf_unused, + $ovfl_unused, + $gap_cnt, + $compressed_size + ); + } +} + +proc integerify {real} { + if {[string is double -strict $real]} { + return [expr {wide($real)}] + } else { + return 0 + } +} +mem function int integerify + +# Quote a string for use in an SQL query. Examples: +# +# [quote {hello world}] == {'hello world'} +# [quote {hello world's}] == {'hello world''s'} +# +proc quote {txt} { + return [string map {' ''} $txt] +} + +# Output a title line +# +proc titleline {title} { + if {$title==""} { + puts [string repeat * 79] + } else { + set len [string length $title] + set stars [string repeat * [expr 79-$len-5]] + puts "*** $title $stars" + } +} + +# Generate a single line of output in the statistics section of the +# report. +# +proc statline {title value {extra {}}} { + set len [string length $title] + set dots [string repeat . [expr 50-$len]] + set len [string length $value] + set sp2 [string range { } $len end] + if {$extra ne ""} { + set extra " $extra" + } + puts "$title$dots $value$sp2$extra" +} + +# Generate a formatted percentage value for $num/$denom +# +proc percent {num denom {of {}}} { + if {$denom==0.0} {return ""} + set v [expr {$num*100.0/$denom}] + set of {} + if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} { + return [format {%5.1f%% %s} $v $of] + } elseif {$v<0.1 || $v>99.9} { + return [format {%7.3f%% %s} $v $of] + } else { + return [format {%6.2f%% %s} $v $of] + } +} + +proc divide {num denom} { + if {$denom==0} {return 0.0} + return [format %.2f [expr double($num)/double($denom)]] +} + +# Generate a subreport that covers some subset of the database. +# the $where clause determines which subset to analyze. +# +proc subreport {title where showFrag} { + global pageSize file_pgcnt compressOverhead + + # Query the in-memory database for the sum of various statistics + # for the subset of tables/indices identified by the WHERE clause in + # $where. Note that even if the WHERE clause matches no rows, the + # following query returns exactly one row (because it is an aggregate). + # + # The results of the query are stored directly by SQLite into local + # variables (i.e. $nentry, $payload etc.). + # + mem eval " + SELECT + int(sum( + CASE WHEN (is_without_rowid OR is_index) THEN nentry + ELSE leaf_entries + END + )) AS nentry, + int(sum(payload)) AS payload, + int(sum(ovfl_payload)) AS ovfl_payload, + max(mx_payload) AS mx_payload, + int(sum(ovfl_cnt)) as ovfl_cnt, + int(sum(leaf_pages)) AS leaf_pages, + int(sum(int_pages)) AS int_pages, + int(sum(ovfl_pages)) AS ovfl_pages, + int(sum(leaf_unused)) AS leaf_unused, + int(sum(int_unused)) AS int_unused, + int(sum(ovfl_unused)) AS ovfl_unused, + int(sum(gap_cnt)) AS gap_cnt, + int(sum(compressed_size)) AS compressed_size, + int(max(depth)) AS depth, + count(*) AS cnt + FROM space_used WHERE $where" {} {} + + # Output the sub-report title, nicely decorated with * characters. + # + puts "" + titleline $title + puts "" + + # Calculate statistics and store the results in TCL variables, as follows: + # + # total_pages: Database pages consumed. + # total_pages_percent: Pages consumed as a percentage of the file. + # storage: Bytes consumed. + # payload_percent: Payload bytes used as a percentage of $storage. + # total_unused: Unused bytes on pages. + # avg_payload: Average payload per btree entry. + # avg_fanout: Average fanout for internal pages. + # avg_unused: Average unused bytes per btree entry. + # avg_meta: Average metadata overhead per entry. + # ovfl_cnt_percent: Percentage of btree entries that use overflow pages. + # + set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] + set total_pages_percent [percent $total_pages $file_pgcnt] + set storage [expr {$total_pages*$pageSize}] + set payload_percent [percent $payload $storage {of storage consumed}] + set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}] + set avg_payload [divide $payload $nentry] + set avg_unused [divide $total_unused $nentry] + set total_meta [expr {$storage - $payload - $total_unused}] + set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}] + set meta_percent [percent $total_meta $storage {of metadata}] + set avg_meta [divide $total_meta $nentry] + if {$int_pages>0} { + # TODO: Is this formula correct? + set nTab [mem eval " + SELECT count(*) FROM ( + SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0 + ) + "] + set avg_fanout [mem eval " + SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used + WHERE $where + "] + set avg_fanout [format %.2f $avg_fanout] + } + set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}] + + # Print out the sub-report statistics. + # + statline {Percentage of total database} $total_pages_percent + statline {Number of entries} $nentry + statline {Bytes of storage consumed} $storage + if {$compressed_size!=$storage} { + set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}] + set pct [expr {$compressed_size*100.0/$storage}] + set pct [format {%5.1f%%} $pct] + statline {Bytes used after compression} $compressed_size $pct + } + statline {Bytes of payload} $payload $payload_percent + statline {Bytes of metadata} $total_meta $meta_percent + if {$cnt==1} {statline {B-tree depth} $depth} + statline {Average payload per entry} $avg_payload + statline {Average unused bytes per entry} $avg_unused + statline {Average metadata per entry} $avg_meta + if {[info exists avg_fanout]} { + statline {Average fanout} $avg_fanout + } + if {$showFrag && $total_pages>1} { + set fragmentation [percent $gap_cnt [expr {$total_pages-1}]] + statline {Non-sequential pages} $gap_cnt $fragmentation + } + statline {Maximum payload per entry} $mx_payload + statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent + if {$int_pages>0} { + statline {Index pages used} $int_pages + } + statline {Primary pages used} $leaf_pages + statline {Overflow pages used} $ovfl_pages + statline {Total pages used} $total_pages + if {$int_unused>0} { + set int_unused_percent [ + percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] + statline "Unused bytes on index pages" $int_unused $int_unused_percent + } + statline "Unused bytes on primary pages" $leaf_unused [ + percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] + statline "Unused bytes on overflow pages" $ovfl_unused [ + percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] + statline "Unused bytes on all pages" $total_unused [ + percent $total_unused $storage {of all space}] + return 1 +} + +# Calculate the overhead in pages caused by auto-vacuum. +# +# This procedure calculates and returns the number of pages used by the +# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum, +# then 0 is returned. The two arguments are the size of the database file in +# pages and the page size used by the database (in bytes). +proc autovacuum_overhead {filePages pageSize} { + + # Set $autovacuum to non-zero for databases that support auto-vacuum. + set autovacuum [db one {PRAGMA auto_vacuum}] + + # If the database is not an auto-vacuum database or the file consists + # of one page only then there is no overhead for auto-vacuum. Return zero. + if {0==$autovacuum || $filePages==1} { + return 0 + } + + # The number of entries on each pointer map page. The layout of the + # database file is one pointer-map page, followed by $ptrsPerPage other + # pages, followed by a pointer-map page etc. The first pointer-map page + # is the second page of the file overall. + set ptrsPerPage [expr double($pageSize/5)] + + # Return the number of pointer map pages in the database. + return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))] +} + + +# Calculate the summary statistics for the database and store the results +# in TCL variables. They are output below. Variables are as follows: +# +# pageSize: Size of each page in bytes. +# file_bytes: File size in bytes. +# file_pgcnt: Number of pages in the file. +# file_pgcnt2: Number of pages in the file (calculated). +# av_pgcnt: Pages consumed by the auto-vacuum pointer-map. +# av_percent: Percentage of the file consumed by auto-vacuum pointer-map. +# inuse_pgcnt: Data pages in the file. +# inuse_percent: Percentage of pages used to store data. +# free_pgcnt: Free pages calculated as ( - ) +# free_pgcnt2: Free pages in the file according to the file header. +# free_percent: Percentage of file consumed by free pages (calculated). +# free_percent2: Percentage of file consumed by free pages (header). +# ntable: Number of tables in the db. +# nindex: Number of indices in the db. +# nautoindex: Number of indices created automatically. +# nmanindex: Number of indices created manually. +# user_payload: Number of bytes of payload in table btrees +# (not including sqlite_master) +# user_percent: $user_payload as a percentage of total file size. + +### The following, setting $file_bytes based on the actual size of the file +### on disk, causes this tool to choke on zipvfs databases. So set it based +### on the return of [PRAGMA page_count] instead. +if 0 { + set file_bytes [file size $file_to_analyze] + set file_pgcnt [expr {$file_bytes/$pageSize}] +} +set file_pgcnt [db one {PRAGMA page_count}] +set file_bytes [expr {$file_pgcnt * $pageSize}] + +set av_pgcnt [autovacuum_overhead $file_pgcnt $pageSize] +set av_percent [percent $av_pgcnt $file_pgcnt] + +set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used} +set inuse_pgcnt [expr wide([mem eval $sql])] +set inuse_percent [percent $inuse_pgcnt $file_pgcnt] + +set free_pgcnt [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}] +set free_percent [percent $free_pgcnt $file_pgcnt] +set free_pgcnt2 [db one {PRAGMA freelist_count}] +set free_percent2 [percent $free_pgcnt2 $file_pgcnt] + +set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] + +set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] +set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] +set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'} +set nautoindex [db eval $sql] +set nmanindex [expr {$nindex-$nautoindex}] + +# set total_payload [mem eval "SELECT sum(payload) FROM space_used"] +set user_payload [mem one {SELECT int(sum(payload)) FROM space_used + WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}] +set user_percent [percent $user_payload $file_bytes] + +# Output the summary statistics calculated above. +# +puts "/** Disk-Space Utilization Report For $root_filename" +puts "" +statline {Page size in bytes} $pageSize +statline {Pages in the whole file (measured)} $file_pgcnt +statline {Pages in the whole file (calculated)} $file_pgcnt2 +statline {Pages that store data} $inuse_pgcnt $inuse_percent +statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 +statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent +statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent +statline {Number of tables in the database} $ntable +statline {Number of indices} $nindex +statline {Number of defined indices} $nmanindex +statline {Number of implied indices} $nautoindex +if {$isCompressed} { + statline {Size of uncompressed content in bytes} $file_bytes + set efficiency [percent $true_file_size $file_bytes] + statline {Size of compressed file on disk} $true_file_size $efficiency +} else { + statline {Size of the file in bytes} $file_bytes +} +statline {Bytes of user payload stored} $user_payload $user_percent + +# Output table rankings +# +puts "" +titleline "Page counts for all tables with their indices" +puts "" +mem eval {SELECT tblname, count(*) AS cnt, + int(sum(int_pages+leaf_pages+ovfl_pages)) AS size + FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} { + statline [string toupper $tblname] $size [percent $size $file_pgcnt] +} +puts "" +titleline "Page counts for all tables and indices separately" +puts "" +mem eval { + SELECT + upper(name) AS nm, + int(int_pages+leaf_pages+ovfl_pages) AS size + FROM space_used + ORDER BY size+0 DESC, name} {} { + statline $nm $size [percent $size $file_pgcnt] +} +if {$isCompressed} { + puts "" + titleline "Bytes of disk space used after compression" + puts "" + set csum 0 + mem eval {SELECT tblname, + int(sum(compressed_size)) + + $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages) + AS csize + FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} { + incr csum $csize + statline [string toupper $tblname] $csize [percent $csize $true_file_size] + } + set overhead [expr {$true_file_size - $csum}] + if {$overhead>0} { + statline {Header and free space} $overhead [percent $overhead $true_file_size] + } +} + +# Output subreports +# +if {$nindex>0} { + subreport {All tables and indices} 1 0 +} +subreport {All tables} {NOT is_index} 0 +if {$nindex>0} { + subreport {All indices} {is_index} 0 +} +foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used + ORDER BY name}] { + set qn [quote $tbl] + set name [string toupper $tbl] + set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}] + if {$n>1} { + set idxlist [mem eval "SELECT name FROM space_used + WHERE tblname='$qn' AND is_index + ORDER BY 1"] + subreport "Table $name and all its indices" "tblname='$qn'" 0 + subreport "Table $name w/o any indices" "name='$qn'" 1 + if {[llength $idxlist]>1} { + subreport "Indices of table $name" "tblname='$qn' AND is_index" 0 + } + foreach idx $idxlist { + set qidx [quote $idx] + subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1 + } + } else { + subreport "Table $name" "name='$qn'" 1 + } +} + +# Output instructions on what the numbers above mean. +# +puts "" +titleline Definitions +puts { +Page size in bytes + + The number of bytes in a single page of the database file. + Usually 1024. + +Number of pages in the whole file +} +puts " The number of $pageSize-byte pages that go into forming the complete + database" +puts { +Pages that store data + + The number of pages that store data, either as primary B*Tree pages or + as overflow pages. The number at the right is the data pages divided by + the total number of pages in the file. + +Pages on the freelist + + The number of pages that are not currently in use but are reserved for + future use. The percentage at the right is the number of freelist pages + divided by the total number of pages in the file. + +Pages of auto-vacuum overhead + + The number of pages that store data used by the database to facilitate + auto-vacuum. This is zero for databases that do not support auto-vacuum. + +Number of tables in the database + + The number of tables in the database, including the SQLITE_MASTER table + used to store schema information. + +Number of indices + + The total number of indices in the database. + +Number of defined indices + + The number of indices created using an explicit CREATE INDEX statement. + +Number of implied indices + + The number of indices used to implement PRIMARY KEY or UNIQUE constraints + on tables. + +Size of the file in bytes + + The total amount of disk space used by the entire database files. + +Bytes of user payload stored + + The total number of bytes of user payload stored in the database. The + schema information in the SQLITE_MASTER table is not counted when + computing this number. The percentage at the right shows the payload + divided by the total file size. + +Percentage of total database + + The amount of the complete database file that is devoted to storing + information described by this category. + +Number of entries + + The total number of B-Tree key/value pairs stored under this category. + +Bytes of storage consumed + + The total amount of disk space required to store all B-Tree entries + under this category. The is the total number of pages used times + the pages size. + +Bytes of payload + + The amount of payload stored under this category. Payload is the data + part of table entries and the key part of index entries. The percentage + at the right is the bytes of payload divided by the bytes of storage + consumed. + +Bytes of metadata + + The amount of formatting and structural information stored in the + table or index. Metadata includes the btree page header, the cell pointer + array, the size field for each cell, the left child pointer or non-leaf + cells, the overflow pointers for overflow cells, and the rowid value for + rowid table cells. In other words, metadata is everything that is neither + unused space nor content. The record header in the payload is counted as + content, not metadata. + +Average payload per entry + + The average amount of payload on each entry. This is just the bytes of + payload divided by the number of entries. + +Average unused bytes per entry + + The average amount of free space remaining on all pages under this + category on a per-entry basis. This is the number of unused bytes on + all pages divided by the number of entries. + +Non-sequential pages + + The number of pages in the table or index that are out of sequence. + Many filesystems are optimized for sequential file access so a small + number of non-sequential pages might result in faster queries, + especially for larger database files that do not fit in the disk cache. + Note that after running VACUUM, the root page of each table or index is + at the beginning of the database file and all other pages are in a + separate part of the database file, resulting in a single non- + sequential page. + +Maximum payload per entry + + The largest payload size of any entry. + +Entries that use overflow + + The number of entries that user one or more overflow pages. + +Total pages used + + This is the number of pages used to hold all information in the current + category. This is the sum of index, primary, and overflow pages. + +Index pages used + + This is the number of pages in a table B-tree that hold only key (rowid) + information and no data. + +Primary pages used + + This is the number of B-tree pages that hold both key and data. + +Overflow pages used + + The total number of overflow pages used for this category. + +Unused bytes on index pages + + The total number of bytes of unused space on all index pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on index pages. + +Unused bytes on primary pages + + The total number of bytes of unused space on all primary pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on primary pages. + +Unused bytes on overflow pages + + The total number of bytes of unused space on all overflow pages. The + percentage at the right is the number of unused bytes divided by the + total number of bytes on overflow pages. + +Unused bytes on all pages + + The total number of bytes of unused space on all primary and overflow + pages. The percentage at the right is the number of unused bytes + divided by the total number of bytes. +} + +# Output a dump of the in-memory database. This can be used for more +# complex offline analysis. +# +titleline {} +puts "The entire text of this report can be sourced into any SQL database" +puts "engine for further analysis. All of the text above is an SQL comment." +puts "The data used to generate this report follows:" +puts "*/" +puts "BEGIN;" +puts $tabledef +unset -nocomplain x +mem eval {SELECT * FROM space_used} x { + puts -nonewline "INSERT INTO space_used VALUES" + set sep ( + foreach col $x(*) { + set v $x($col) + if {$v=="" || ![string is double $v]} {set v '[quote $v]'} + puts -nonewline $sep$v + set sep , + } + puts ");" +} +puts "COMMIT;" + +} err]} { + puts "ERROR: $err" + puts $errorInfo + exit 1 +} +END_STRING +; +} diff --git a/tool/tostr.tcl b/tool/tostr.tcl deleted file mode 100644 index cb06ee947f..0000000000 --- a/tool/tostr.tcl +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/tcl -# -# Convert input text into a C string -# -set in [open [lindex $argv 0] rb] -while {![eof $in]} { - set line [gets $in] - if {[eof $in]} break; - set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] - puts "\"$x\\n\"" -} -close $in From 96a206fa1055e1f6fa3d0bf8a221bdf18bc7195d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 13 Oct 2017 20:14:06 +0000 Subject: [PATCH 016/156] Improved ability to generate stand-alone program using TCL and SQLite by compiling with -DTCLSH_INIT_PROC=name to cause the TCL interpreter to be initialized using procedure name(). Both sqlite3_analyzer and testfixture are now built this way. FossilOrigin-Name: d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 --- Makefile.in | 5 +- Makefile.msc | 4 +- main.mk | 9 +- manifest | 22 +- manifest.uuid | 2 +- src/tclsqlite.c | 138 +++--- src/test_tclsh.c | 194 ++++---- tool/sqlite3_analyzer.c.in | 886 +------------------------------------ 8 files changed, 187 insertions(+), 1073 deletions(-) diff --git a/Makefile.in b/Makefile.in index e2934d80b1..5f6be57c37 100644 --- a/Makefile.in +++ b/Makefile.in @@ -947,7 +947,7 @@ tclsqlite.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c + $(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c @@ -1113,7 +1113,8 @@ sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 diff --git a/Makefile.msc b/Makefile.msc index bfd368c558..ce867ba3f3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1934,7 +1934,7 @@ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) @@ -2105,7 +2105,7 @@ sqlite3rbu.lo: $(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR) # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 diff --git a/main.mk b/main.mk index 18d7cd3459..c7711fb6e9 100644 --- a/main.mk +++ b/main.mk @@ -777,7 +777,7 @@ sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ + $(TCCX) $(TCL_FLAGS) -DTCLSH -o tclsqlite3 \ $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl @@ -798,21 +798,22 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c \ $(TOP)/ext/session/test_session.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ $(TOP)/ext/session/test_session.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ + $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) diff --git a/manifest b/manifest index 669c1b284a..2599e89765 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram. -D 2017-10-13T18:58:55.897 -F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082 +C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway. +D 2017-10-13T20:14:06.865 +F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277 +F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026 +F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4 +F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b @@ -514,7 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 -F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87 +F src/test_tclsh.c 74fcfb7f3b0ff1f871e62263dd84ffba46a8e9d477439115e0fb2035e4bf69e1 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e @@ -1632,7 +1632,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb -F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9 +F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d -R 78f34b33921be1c7a7c8587f43b33b81 +P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 +R df005e0c0c347d89af378f70d82f057b U drh -Z 180ea09ee34fefc9e97e7461c13719ef +Z 8b50e8b5594af446543ab2fc55fee9a2 diff --git a/manifest.uuid b/manifest.uuid index fabaeec3c0..d905d370b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 \ No newline at end of file +d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 0009eab695..eed86eee34 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -14,17 +14,19 @@ ** ** Compile-time options: ** -** -DTCLSH=1 Add a "main()" routine that works as a tclsh. +** -DTCLSH Add a "main()" routine that works as a tclsh. ** -** -DSQLITE_TCLMD5 When used in conjuction with -DTCLSH=1, add -** four new commands to the TCL interpreter for -** generating MD5 checksums: md5, md5file, -** md5-10x8, and md5file-10x8. +** -DTCLSH_INIT_PROC=name ** -** -DSQLITE_TEST When used in conjuction with -DTCLSH=1, add -** hundreds of new commands used for testing -** SQLite. This option implies -DSQLITE_TCLMD5. +** Invoke name(interp) to initialize the Tcl interpreter. +** If name(interp) returns a non-NULL string, then run +** that string as a Tcl script to launch the application. +** If name(interp) returns NULL, then run the regular +** tclsh-emulator code. */ +#ifdef TCLSH_INIT_PROC +# define TCLSH 1 +#endif /* ** If requested, include the SQLite compiler options file for MSVC. @@ -3582,56 +3584,55 @@ int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } #endif /* -** If the TCLSH macro is defined to be either 1 or 2, then a main() -** routine is inserted that starts up a Tcl interpreter. When TCLSH==1 -** the interpreter works like an ordinary tclsh. When TCLSH==2 then the -** startup script is supplied by an routine named "tclsh_main_loop()" -** that must be linked separately. The TCLSH==2 technique is used to -** generate stand-alone executables based on TCL, such as -** sqlite3_analyzer.exe. +** If the TCLSH macro is defined, add code to make a stand-alone program. */ -#ifdef TCLSH +#if defined(TCLSH) -/* -** If the macro TCLSH is one, then put in code this for the -** "main" routine that will initialize Tcl and take input from -** standard input, or if a file is named on the command line -** the TCL interpreter reads and evaluates that file. +/* This is the main routine for an ordinary TCL shell. If there are +** are arguments, run the first argument as a script. Otherwise, +** read TCL commands from standard input */ -#if TCLSH==1 static const char *tclsh_main_loop(void){ static const char zMainloop[] = - "set line {}\n" - "while {![eof stdin]} {\n" - "if {$line!=\"\"} {\n" - "puts -nonewline \"> \"\n" - "} else {\n" - "puts -nonewline \"% \"\n" - "}\n" - "flush stdout\n" - "append line [gets stdin]\n" - "if {[info complete $line]} {\n" - "if {[catch {uplevel #0 $line} result]} {\n" - "puts stderr \"Error: $result\"\n" - "} elseif {$result!=\"\"} {\n" - "puts $result\n" + "if {[llength $argv]>=1} {\n" + "set argv0 [lindex $argv 0]\n" + "set argv [lrange $argv 1 end]\n" + "source $argv0\n" + "} else {\n" + "set line {}\n" + "while {![eof stdin]} {\n" + "if {$line!=\"\"} {\n" + "puts -nonewline \"> \"\n" + "} else {\n" + "puts -nonewline \"% \"\n" + "}\n" + "flush stdout\n" + "append line [gets stdin]\n" + "if {[info complete $line]} {\n" + "if {[catch {uplevel #0 $line} result]} {\n" + "puts stderr \"Error: $result\"\n" + "} elseif {$result!=\"\"} {\n" + "puts $result\n" + "}\n" + "set line {}\n" + "} else {\n" + "append line \\n\n" "}\n" - "set line {}\n" - "} else {\n" - "append line \\n\n" "}\n" "}\n" ; return zMainloop; } -#endif -#if TCLSH==2 -static const char *tclsh_main_loop(void); -#endif #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; + int i; + const char *zScript = 0; + char zArgc[32]; +#if defined(TCLSH_INIT_PROC) + extern const char *TCLSH_INIT_PROC(Tcl_Interp*); +#endif #if !defined(_WIN32_WCE) if( getenv("BREAK") ){ @@ -3650,42 +3651,27 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){ Tcl_FindExecutable(argv[0]); Tcl_SetSystemEncoding(NULL, "utf-8"); interp = Tcl_CreateInterp(); - -#if TCLSH==2 - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); -#endif - - /* Add extensions */ -#if !defined(SQLITE_TEST) - /* Normally we only initialize the TCL extension */ Sqlite3_Init(interp); -#else - /* For testing, do lots of extra initialization */ - { - extern void sqlite3InitTclTestLogic(Tcl_Interp*); - sqlite3InitTclTestLogic(interp); + + sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-1); + Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY); + Tcl_SetVar(interp,"argv0",argv[0],TCL_GLOBAL_ONLY); + Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY); + for(i=1; i=2 ){ - int i; - char zArgc[32]; - sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-(3-TCLSH)); - Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY); - Tcl_SetVar(interp,"argv0",argv[1],TCL_GLOBAL_ONLY); - Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY); - for(i=3-TCLSH; i " - } else { - puts -nonewline "% " - } - flush stdout - append line [gets stdin] - if {[info complete $line]} { - if {[catch {uplevel #0 $line} result]} { - puts stderr "Error: $result" - } elseif {$result!=""} { - puts $result - } - set line {} - } else { - append line \n - } - } -} - - -# Get the name of the database to analyze -# -proc usage {} { - set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" - puts stderr { -Analyze the SQLite3 database file specified by the "database-filename" -argument and output a report detailing size and storage efficiency -information for the database and its constituent tables and indexes. - -Options: - - --pageinfo Show how each page of the database-file is used - - --stats Output SQL text that creates a new database containing - statistics about the database that was analyzed - - --tclsh Run the built-in TCL interpreter interactively (for debugging) - - --version Show the version number of SQLite -} - exit 1 -} -set file_to_analyze {} -set flags(-pageinfo) 0 -set flags(-stats) 0 -set flags(-debug) 0 -append argv {} -foreach arg $argv { - if {[regexp {^-+pageinfo$} $arg]} { - set flags(-pageinfo) 1 - } elseif {[regexp {^-+stats$} $arg]} { - set flags(-stats) 1 - } elseif {[regexp {^-+debug$} $arg]} { - set flags(-debug) 1 - } elseif {[regexp {^-+tclsh$} $arg]} { - tclsh - exit 0 - } elseif {[regexp {^-+version$} $arg]} { - sqlite3 mem :memory: - puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] - mem close - exit 0 - } elseif {[regexp {^-} $arg]} { - puts stderr "Unknown option: $arg" - usage - } elseif {$file_to_analyze!=""} { - usage - } else { - set file_to_analyze $arg - } -} -if {$file_to_analyze==""} usage -set root_filename $file_to_analyze -regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename -if {![file exists $root_filename]} { - puts stderr "No such file: $root_filename" - exit 1 -} -if {![file readable $root_filename]} { - puts stderr "File is not readable: $root_filename" - exit 1 -} -set true_file_size [file size $root_filename] -if {$true_file_size<512} { - puts stderr "Empty or malformed database: $root_filename" - exit 1 -} - -# Compute the total file size assuming test_multiplexor is being used. -# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled -# -set extension [file extension $root_filename] -set pattern $root_filename -append pattern {[0-3][0-9][0-9]} -foreach f [glob -nocomplain $pattern] { - incr true_file_size [file size $f] - set extension {} -} -if {[string length $extension]>=2 && [string length $extension]<=4} { - set pattern [file rootname $root_filename] - append pattern {.[0-3][0-9][0-9]} - foreach f [glob -nocomplain $pattern] { - incr true_file_size [file size $f] - } -} - -# Open the database -# -if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} { - puts stderr "error trying to open $file_to_analyze: $msg" - exit 1 -} -if {$flags(-debug)} { - proc dbtrace {txt} {puts $txt; flush stdout;} - db trace ::dbtrace -} - -db eval {SELECT count(*) FROM sqlite_master} -set pageSize [expr {wide([db one {PRAGMA page_size}])}] - -if {$flags(-pageinfo)} { - db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} - db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} { - puts "$pageno $name $path" - } - exit 0 -} -if {$flags(-stats)} { - db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} - puts "BEGIN;" - puts "CREATE TABLE stats(" - puts " name STRING, /* Name of table or index */" - puts " path INTEGER, /* Path to page from root */" - puts " pageno INTEGER, /* Page number */" - puts " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" - puts " ncell INTEGER, /* Cells on page (0 for overflow) */" - puts " payload INTEGER, /* Bytes of payload on this page */" - puts " unused INTEGER, /* Bytes of unused space on this page */" - puts " mx_payload INTEGER, /* Largest payload size of all cells */" - puts " pgoffset INTEGER, /* Offset of page in file */" - puts " pgsize INTEGER /* Size of the page */" - puts ");" - db eval {SELECT quote(name) || ',' || - quote(path) || ',' || - quote(pageno) || ',' || - quote(pagetype) || ',' || - quote(ncell) || ',' || - quote(payload) || ',' || - quote(unused) || ',' || - quote(mx_payload) || ',' || - quote(pgoffset) || ',' || - quote(pgsize) AS x FROM stat} { - puts "INSERT INTO stats VALUES($x);" - } - puts "COMMIT;" - exit 0 -} - - -# In-memory database for collecting statistics. This script loops through -# the tables and indices in the database being analyzed, adding a row for each -# to an in-memory database (for which the schema is shown below). It then -# queries the in-memory db to produce the space-analysis report. -# -sqlite3 mem :memory: -if {$flags(-debug)} { - proc dbtrace {txt} {puts $txt; flush stdout;} - mem trace ::dbtrace -} -set tabledef {CREATE TABLE space_used( - name clob, -- Name of a table or index in the database file - tblname clob, -- Name of associated table - is_index boolean, -- TRUE if it is an index, false for a table - is_without_rowid boolean, -- TRUE if WITHOUT ROWID table - nentry int, -- Number of entries in the BTree - leaf_entries int, -- Number of leaf entries - depth int, -- Depth of the b-tree - payload int, -- Total amount of data stored in this table or index - ovfl_payload int, -- Total amount of data stored on overflow pages - ovfl_cnt int, -- Number of entries that use overflow - mx_payload int, -- Maximum payload size - int_pages int, -- Number of interior pages used - leaf_pages int, -- Number of leaf pages used - ovfl_pages int, -- Number of overflow pages used - int_unused int, -- Number of unused bytes on interior pages - leaf_unused int, -- Number of unused bytes on primary pages - ovfl_unused int, -- Number of unused bytes on overflow pages - gap_cnt int, -- Number of gaps in the page layout - compressed_size int -- Total bytes stored on disk -);} -mem eval $tabledef - -# Create a temporary "dbstat" virtual table. -# -db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat} -db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat - ORDER BY name, path} -db eval {DROP TABLE temp.stat} - -set isCompressed 0 -set compressOverhead 0 -set depth 0 -set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } -foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { - - set is_index [expr {$name!=$tblname}] - set is_without_rowid [is_without_rowid $name] - db eval { - SELECT - sum(ncell) AS nentry, - sum((pagetype=='leaf')*ncell) AS leaf_entries, - sum(payload) AS payload, - sum((pagetype=='overflow') * payload) AS ovfl_payload, - sum(path LIKE '%+000000') AS ovfl_cnt, - max(mx_payload) AS mx_payload, - sum(pagetype=='internal') AS int_pages, - sum(pagetype=='leaf') AS leaf_pages, - sum(pagetype=='overflow') AS ovfl_pages, - sum((pagetype=='internal') * unused) AS int_unused, - sum((pagetype=='leaf') * unused) AS leaf_unused, - sum((pagetype=='overflow') * unused) AS ovfl_unused, - sum(pgsize) AS compressed_size, - max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4) - AS depth - FROM temp.dbstat WHERE name = $name - } break - - set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] - set storage [expr {$total_pages*$pageSize}] - if {!$isCompressed && $storage>$compressed_size} { - set isCompressed 1 - set compressOverhead 14 - } - - # Column 'gap_cnt' is set to the number of non-contiguous entries in the - # list of pages visited if the b-tree structure is traversed in a top-down - # fashion (each node visited before its child-tree is passed). Any overflow - # chains present are traversed from start to finish before any child-tree - # is. - # - set gap_cnt 0 - set prev 0 - db eval { - SELECT pageno, pagetype FROM temp.dbstat - WHERE name=$name - ORDER BY pageno - } { - if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} { - incr gap_cnt - } - set prev $pageno - } - mem eval { - INSERT INTO space_used VALUES( - $name, - $tblname, - $is_index, - $is_without_rowid, - $nentry, - $leaf_entries, - $depth, - $payload, - $ovfl_payload, - $ovfl_cnt, - $mx_payload, - $int_pages, - $leaf_pages, - $ovfl_pages, - $int_unused, - $leaf_unused, - $ovfl_unused, - $gap_cnt, - $compressed_size - ); - } -} - -proc integerify {real} { - if {[string is double -strict $real]} { - return [expr {wide($real)}] - } else { - return 0 - } -} -mem function int integerify - -# Quote a string for use in an SQL query. Examples: -# -# [quote {hello world}] == {'hello world'} -# [quote {hello world's}] == {'hello world''s'} -# -proc quote {txt} { - return [string map {' ''} $txt] -} - -# Output a title line -# -proc titleline {title} { - if {$title==""} { - puts [string repeat * 79] - } else { - set len [string length $title] - set stars [string repeat * [expr 79-$len-5]] - puts "*** $title $stars" - } -} - -# Generate a single line of output in the statistics section of the -# report. -# -proc statline {title value {extra {}}} { - set len [string length $title] - set dots [string repeat . [expr 50-$len]] - set len [string length $value] - set sp2 [string range { } $len end] - if {$extra ne ""} { - set extra " $extra" - } - puts "$title$dots $value$sp2$extra" -} - -# Generate a formatted percentage value for $num/$denom -# -proc percent {num denom {of {}}} { - if {$denom==0.0} {return ""} - set v [expr {$num*100.0/$denom}] - set of {} - if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} { - return [format {%5.1f%% %s} $v $of] - } elseif {$v<0.1 || $v>99.9} { - return [format {%7.3f%% %s} $v $of] - } else { - return [format {%6.2f%% %s} $v $of] - } -} - -proc divide {num denom} { - if {$denom==0} {return 0.0} - return [format %.2f [expr double($num)/double($denom)]] -} - -# Generate a subreport that covers some subset of the database. -# the $where clause determines which subset to analyze. -# -proc subreport {title where showFrag} { - global pageSize file_pgcnt compressOverhead - - # Query the in-memory database for the sum of various statistics - # for the subset of tables/indices identified by the WHERE clause in - # $where. Note that even if the WHERE clause matches no rows, the - # following query returns exactly one row (because it is an aggregate). - # - # The results of the query are stored directly by SQLite into local - # variables (i.e. $nentry, $payload etc.). - # - mem eval " - SELECT - int(sum( - CASE WHEN (is_without_rowid OR is_index) THEN nentry - ELSE leaf_entries - END - )) AS nentry, - int(sum(payload)) AS payload, - int(sum(ovfl_payload)) AS ovfl_payload, - max(mx_payload) AS mx_payload, - int(sum(ovfl_cnt)) as ovfl_cnt, - int(sum(leaf_pages)) AS leaf_pages, - int(sum(int_pages)) AS int_pages, - int(sum(ovfl_pages)) AS ovfl_pages, - int(sum(leaf_unused)) AS leaf_unused, - int(sum(int_unused)) AS int_unused, - int(sum(ovfl_unused)) AS ovfl_unused, - int(sum(gap_cnt)) AS gap_cnt, - int(sum(compressed_size)) AS compressed_size, - int(max(depth)) AS depth, - count(*) AS cnt - FROM space_used WHERE $where" {} {} - - # Output the sub-report title, nicely decorated with * characters. - # - puts "" - titleline $title - puts "" - - # Calculate statistics and store the results in TCL variables, as follows: - # - # total_pages: Database pages consumed. - # total_pages_percent: Pages consumed as a percentage of the file. - # storage: Bytes consumed. - # payload_percent: Payload bytes used as a percentage of $storage. - # total_unused: Unused bytes on pages. - # avg_payload: Average payload per btree entry. - # avg_fanout: Average fanout for internal pages. - # avg_unused: Average unused bytes per btree entry. - # avg_meta: Average metadata overhead per entry. - # ovfl_cnt_percent: Percentage of btree entries that use overflow pages. - # - set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}] - set total_pages_percent [percent $total_pages $file_pgcnt] - set storage [expr {$total_pages*$pageSize}] - set payload_percent [percent $payload $storage {of storage consumed}] - set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}] - set avg_payload [divide $payload $nentry] - set avg_unused [divide $total_unused $nentry] - set total_meta [expr {$storage - $payload - $total_unused}] - set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}] - set meta_percent [percent $total_meta $storage {of metadata}] - set avg_meta [divide $total_meta $nentry] - if {$int_pages>0} { - # TODO: Is this formula correct? - set nTab [mem eval " - SELECT count(*) FROM ( - SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0 - ) - "] - set avg_fanout [mem eval " - SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used - WHERE $where - "] - set avg_fanout [format %.2f $avg_fanout] - } - set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}] - - # Print out the sub-report statistics. - # - statline {Percentage of total database} $total_pages_percent - statline {Number of entries} $nentry - statline {Bytes of storage consumed} $storage - if {$compressed_size!=$storage} { - set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}] - set pct [expr {$compressed_size*100.0/$storage}] - set pct [format {%5.1f%%} $pct] - statline {Bytes used after compression} $compressed_size $pct - } - statline {Bytes of payload} $payload $payload_percent - statline {Bytes of metadata} $total_meta $meta_percent - if {$cnt==1} {statline {B-tree depth} $depth} - statline {Average payload per entry} $avg_payload - statline {Average unused bytes per entry} $avg_unused - statline {Average metadata per entry} $avg_meta - if {[info exists avg_fanout]} { - statline {Average fanout} $avg_fanout - } - if {$showFrag && $total_pages>1} { - set fragmentation [percent $gap_cnt [expr {$total_pages-1}]] - statline {Non-sequential pages} $gap_cnt $fragmentation - } - statline {Maximum payload per entry} $mx_payload - statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent - if {$int_pages>0} { - statline {Index pages used} $int_pages - } - statline {Primary pages used} $leaf_pages - statline {Overflow pages used} $ovfl_pages - statline {Total pages used} $total_pages - if {$int_unused>0} { - set int_unused_percent [ - percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] - statline "Unused bytes on index pages" $int_unused $int_unused_percent - } - statline "Unused bytes on primary pages" $leaf_unused [ - percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] - statline "Unused bytes on overflow pages" $ovfl_unused [ - percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] - statline "Unused bytes on all pages" $total_unused [ - percent $total_unused $storage {of all space}] - return 1 -} - -# Calculate the overhead in pages caused by auto-vacuum. -# -# This procedure calculates and returns the number of pages used by the -# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum, -# then 0 is returned. The two arguments are the size of the database file in -# pages and the page size used by the database (in bytes). -proc autovacuum_overhead {filePages pageSize} { - - # Set $autovacuum to non-zero for databases that support auto-vacuum. - set autovacuum [db one {PRAGMA auto_vacuum}] - - # If the database is not an auto-vacuum database or the file consists - # of one page only then there is no overhead for auto-vacuum. Return zero. - if {0==$autovacuum || $filePages==1} { - return 0 - } - - # The number of entries on each pointer map page. The layout of the - # database file is one pointer-map page, followed by $ptrsPerPage other - # pages, followed by a pointer-map page etc. The first pointer-map page - # is the second page of the file overall. - set ptrsPerPage [expr double($pageSize/5)] - - # Return the number of pointer map pages in the database. - return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))] -} - - -# Calculate the summary statistics for the database and store the results -# in TCL variables. They are output below. Variables are as follows: -# -# pageSize: Size of each page in bytes. -# file_bytes: File size in bytes. -# file_pgcnt: Number of pages in the file. -# file_pgcnt2: Number of pages in the file (calculated). -# av_pgcnt: Pages consumed by the auto-vacuum pointer-map. -# av_percent: Percentage of the file consumed by auto-vacuum pointer-map. -# inuse_pgcnt: Data pages in the file. -# inuse_percent: Percentage of pages used to store data. -# free_pgcnt: Free pages calculated as ( - ) -# free_pgcnt2: Free pages in the file according to the file header. -# free_percent: Percentage of file consumed by free pages (calculated). -# free_percent2: Percentage of file consumed by free pages (header). -# ntable: Number of tables in the db. -# nindex: Number of indices in the db. -# nautoindex: Number of indices created automatically. -# nmanindex: Number of indices created manually. -# user_payload: Number of bytes of payload in table btrees -# (not including sqlite_master) -# user_percent: $user_payload as a percentage of total file size. - -### The following, setting $file_bytes based on the actual size of the file -### on disk, causes this tool to choke on zipvfs databases. So set it based -### on the return of [PRAGMA page_count] instead. -if 0 { - set file_bytes [file size $file_to_analyze] - set file_pgcnt [expr {$file_bytes/$pageSize}] -} -set file_pgcnt [db one {PRAGMA page_count}] -set file_bytes [expr {$file_pgcnt * $pageSize}] - -set av_pgcnt [autovacuum_overhead $file_pgcnt $pageSize] -set av_percent [percent $av_pgcnt $file_pgcnt] - -set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used} -set inuse_pgcnt [expr wide([mem eval $sql])] -set inuse_percent [percent $inuse_pgcnt $file_pgcnt] - -set free_pgcnt [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}] -set free_percent [percent $free_pgcnt $file_pgcnt] -set free_pgcnt2 [db one {PRAGMA freelist_count}] -set free_percent2 [percent $free_pgcnt2 $file_pgcnt] - -set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] - -set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] -set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] -set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'} -set nautoindex [db eval $sql] -set nmanindex [expr {$nindex-$nautoindex}] - -# set total_payload [mem eval "SELECT sum(payload) FROM space_used"] -set user_payload [mem one {SELECT int(sum(payload)) FROM space_used - WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}] -set user_percent [percent $user_payload $file_bytes] - -# Output the summary statistics calculated above. -# -puts "/** Disk-Space Utilization Report For $root_filename" -puts "" -statline {Page size in bytes} $pageSize -statline {Pages in the whole file (measured)} $file_pgcnt -statline {Pages in the whole file (calculated)} $file_pgcnt2 -statline {Pages that store data} $inuse_pgcnt $inuse_percent -statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 -statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent -statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent -statline {Number of tables in the database} $ntable -statline {Number of indices} $nindex -statline {Number of defined indices} $nmanindex -statline {Number of implied indices} $nautoindex -if {$isCompressed} { - statline {Size of uncompressed content in bytes} $file_bytes - set efficiency [percent $true_file_size $file_bytes] - statline {Size of compressed file on disk} $true_file_size $efficiency -} else { - statline {Size of the file in bytes} $file_bytes -} -statline {Bytes of user payload stored} $user_payload $user_percent - -# Output table rankings -# -puts "" -titleline "Page counts for all tables with their indices" -puts "" -mem eval {SELECT tblname, count(*) AS cnt, - int(sum(int_pages+leaf_pages+ovfl_pages)) AS size - FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} { - statline [string toupper $tblname] $size [percent $size $file_pgcnt] -} -puts "" -titleline "Page counts for all tables and indices separately" -puts "" -mem eval { - SELECT - upper(name) AS nm, - int(int_pages+leaf_pages+ovfl_pages) AS size - FROM space_used - ORDER BY size+0 DESC, name} {} { - statline $nm $size [percent $size $file_pgcnt] -} -if {$isCompressed} { - puts "" - titleline "Bytes of disk space used after compression" - puts "" - set csum 0 - mem eval {SELECT tblname, - int(sum(compressed_size)) + - $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages) - AS csize - FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} { - incr csum $csize - statline [string toupper $tblname] $csize [percent $csize $true_file_size] - } - set overhead [expr {$true_file_size - $csum}] - if {$overhead>0} { - statline {Header and free space} $overhead [percent $overhead $true_file_size] - } -} - -# Output subreports -# -if {$nindex>0} { - subreport {All tables and indices} 1 0 -} -subreport {All tables} {NOT is_index} 0 -if {$nindex>0} { - subreport {All indices} {is_index} 0 -} -foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used - ORDER BY name}] { - set qn [quote $tbl] - set name [string toupper $tbl] - set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}] - if {$n>1} { - set idxlist [mem eval "SELECT name FROM space_used - WHERE tblname='$qn' AND is_index - ORDER BY 1"] - subreport "Table $name and all its indices" "tblname='$qn'" 0 - subreport "Table $name w/o any indices" "name='$qn'" 1 - if {[llength $idxlist]>1} { - subreport "Indices of table $name" "tblname='$qn' AND is_index" 0 - } - foreach idx $idxlist { - set qidx [quote $idx] - subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1 - } - } else { - subreport "Table $name" "name='$qn'" 1 - } -} - -# Output instructions on what the numbers above mean. -# -puts "" -titleline Definitions -puts { -Page size in bytes - - The number of bytes in a single page of the database file. - Usually 1024. - -Number of pages in the whole file -} -puts " The number of $pageSize-byte pages that go into forming the complete - database" -puts { -Pages that store data - - The number of pages that store data, either as primary B*Tree pages or - as overflow pages. The number at the right is the data pages divided by - the total number of pages in the file. - -Pages on the freelist - - The number of pages that are not currently in use but are reserved for - future use. The percentage at the right is the number of freelist pages - divided by the total number of pages in the file. - -Pages of auto-vacuum overhead - - The number of pages that store data used by the database to facilitate - auto-vacuum. This is zero for databases that do not support auto-vacuum. - -Number of tables in the database - - The number of tables in the database, including the SQLITE_MASTER table - used to store schema information. - -Number of indices - - The total number of indices in the database. - -Number of defined indices - - The number of indices created using an explicit CREATE INDEX statement. - -Number of implied indices - - The number of indices used to implement PRIMARY KEY or UNIQUE constraints - on tables. - -Size of the file in bytes - - The total amount of disk space used by the entire database files. - -Bytes of user payload stored - - The total number of bytes of user payload stored in the database. The - schema information in the SQLITE_MASTER table is not counted when - computing this number. The percentage at the right shows the payload - divided by the total file size. - -Percentage of total database - - The amount of the complete database file that is devoted to storing - information described by this category. - -Number of entries - - The total number of B-Tree key/value pairs stored under this category. - -Bytes of storage consumed - - The total amount of disk space required to store all B-Tree entries - under this category. The is the total number of pages used times - the pages size. - -Bytes of payload - - The amount of payload stored under this category. Payload is the data - part of table entries and the key part of index entries. The percentage - at the right is the bytes of payload divided by the bytes of storage - consumed. - -Bytes of metadata - - The amount of formatting and structural information stored in the - table or index. Metadata includes the btree page header, the cell pointer - array, the size field for each cell, the left child pointer or non-leaf - cells, the overflow pointers for overflow cells, and the rowid value for - rowid table cells. In other words, metadata is everything that is neither - unused space nor content. The record header in the payload is counted as - content, not metadata. - -Average payload per entry - - The average amount of payload on each entry. This is just the bytes of - payload divided by the number of entries. - -Average unused bytes per entry - - The average amount of free space remaining on all pages under this - category on a per-entry basis. This is the number of unused bytes on - all pages divided by the number of entries. - -Non-sequential pages - - The number of pages in the table or index that are out of sequence. - Many filesystems are optimized for sequential file access so a small - number of non-sequential pages might result in faster queries, - especially for larger database files that do not fit in the disk cache. - Note that after running VACUUM, the root page of each table or index is - at the beginning of the database file and all other pages are in a - separate part of the database file, resulting in a single non- - sequential page. - -Maximum payload per entry - - The largest payload size of any entry. - -Entries that use overflow - - The number of entries that user one or more overflow pages. - -Total pages used - - This is the number of pages used to hold all information in the current - category. This is the sum of index, primary, and overflow pages. - -Index pages used - - This is the number of pages in a table B-tree that hold only key (rowid) - information and no data. - -Primary pages used - - This is the number of B-tree pages that hold both key and data. - -Overflow pages used - - The total number of overflow pages used for this category. - -Unused bytes on index pages - - The total number of bytes of unused space on all index pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on index pages. - -Unused bytes on primary pages - - The total number of bytes of unused space on all primary pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on primary pages. - -Unused bytes on overflow pages - - The total number of bytes of unused space on all overflow pages. The - percentage at the right is the number of unused bytes divided by the - total number of bytes on overflow pages. - -Unused bytes on all pages - - The total number of bytes of unused space on all primary and overflow - pages. The percentage at the right is the number of unused bytes - divided by the total number of bytes. -} - -# Output a dump of the in-memory database. This can be used for more -# complex offline analysis. -# -titleline {} -puts "The entire text of this report can be sourced into any SQL database" -puts "engine for further analysis. All of the text above is an SQL comment." -puts "The data used to generate this report follows:" -puts "*/" -puts "BEGIN;" -puts $tabledef -unset -nocomplain x -mem eval {SELECT * FROM space_used} x { - puts -nonewline "INSERT INTO space_used VALUES" - set sep ( - foreach col $x(*) { - set v $x($col) - if {$v=="" || ![string is double $v]} {set v '[quote $v]'} - puts -nonewline $sep$v - set sep , - } - puts ");" -} -puts "COMMIT;" - -} err]} { - puts "ERROR: $err" - puts $errorInfo - exit 1 -} +INCLUDE $ROOT/tool/spaceanal.tcl END_STRING ; } From 835cd436e1e0a964d84034a88303a1b4c0e8083c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 14 Oct 2017 19:54:06 +0000 Subject: [PATCH 017/156] Updates to the Makefiles for MSVC. FossilOrigin-Name: ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 --- Makefile.msc | 4 ++-- autoconf/Makefile.msc | 5 +++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index ce867ba3f3..2804c25ab4 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1984,8 +1984,8 @@ SHELL_SRC = \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\completion.c -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl - $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c +shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl + $(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c # Rules to build the extension objects. diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index f535d1450a..e0f7ad8615 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -928,6 +928,7 @@ LIBRESOBJS = # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB !ENDIF @@ -957,8 +958,8 @@ sqlite3.def: Replace.exe $(LIBOBJ) | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def -$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) - $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ +$(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index 2599e89765..1ce91e738e 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway. -D 2017-10-13T20:14:06.865 +C Updates\sto\sthe\sMakefiles\sfor\sMSVC. +D 2017-10-14T19:54:06.773 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4 +F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd +F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82 -R df005e0c0c347d89af378f70d82f057b -U drh -Z 8b50e8b5594af446543ab2fc55fee9a2 +P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 +R 49fa7e54acb9ad8a392e6c25fe8116d5 +U mistachkin +Z 3270bf7f35234fd560a264ffe40cbf35 diff --git a/manifest.uuid b/manifest.uuid index d905d370b6..63b3b0cf2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 \ No newline at end of file +ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 \ No newline at end of file From 82cae9ffd62c1375adef56d9e0d43c099691c797 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 14 Oct 2017 19:58:37 +0000 Subject: [PATCH 018/156] Updates to the Makefiles for MSVC. Cherrypick of [ac8786f3f9f35cb6]. FossilOrigin-Name: 92eb721faefcdd8396072722d3e4d7ca41b860b306e4bb0f0191dde8f30d0add --- Makefile.msc | 4 ++-- autoconf/Makefile.msc | 4 ++-- manifest | 17 +++++++++-------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 4c3c67f94d..2e18c50d47 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1976,8 +1976,8 @@ SHELL_SRC = \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\completion.c -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl - $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c +shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl + $(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c # Rules to build the extension objects. diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index f535d1450a..46f7c7a25b 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -957,8 +957,8 @@ sqlite3.def: Replace.exe $(LIBOBJ) | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def -$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) - $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ +$(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) diff --git a/manifest b/manifest index b720d2abb5..9e350ba263 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sa\sharmless\scompiler\swarning\sabout\san\sunused\sparameter. -D 2017-10-12T14:18:38.182 +C Updates\sto\sthe\sMakefiles\sfor\sMSVC.\s\sCherrypick\sof\s[ac8786f3f9f35cb6]. +D 2017-10-14T19:58:37.221 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20 +F Makefile.msc 4a7ebdaacec1abd415d2847c8f5b42053c5963294e925672d43d2972a5bb2cb6 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 -F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd +F autoconf/Makefile.msc b045158822d2320d5551cb7291ecc685dd6123324b928490691deed71274c305 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 @@ -1656,7 +1656,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339 -R 6b8c8a980bca85216c2bbb28216f8b36 -U drh -Z ee2d81a451eaa5a0334a9c3d00838821 +P de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0 +Q +ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 +R 8e422c1ee06cd57625ef547ccd862b5a +U mistachkin +Z 0ed36b5b53069fc1c0ccb6054caf79bd diff --git a/manifest.uuid b/manifest.uuid index 0b314a09be..524d43870c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0 \ No newline at end of file +92eb721faefcdd8396072722d3e4d7ca41b860b306e4bb0f0191dde8f30d0add \ No newline at end of file From 47b1d68f25f2c921497c6a642a111d2d8e7d00cc Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 15 Oct 2017 22:16:25 +0000 Subject: [PATCH 019/156] In the query planner, do not consider index X to be a proper subset of index Y if X is a covering index but Y is not. FossilOrigin-Name: ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 17 ++++++++++------- test/analyze9.test | 5 ++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 1ce91e738e..b77b1484ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sMakefiles\sfor\sMSVC. -D 2017-10-14T19:54:06.773 +C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot. +D 2017-10-15T22:16:25.229 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 -F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 +F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 @@ -568,7 +568,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 -F test/analyze9.test b817b8e798315fc65b820a5463f73ad5f48ed8dd +F test/analyze9.test dac0bdc7eab965b9ad639ca83564d98717aaf13ce5a776f23d9a3680238cecd8 F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test a4c1c3048f6d9e090eb76e83eecb18bcf6d31a70 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4 -R 49fa7e54acb9ad8a392e6c25fe8116d5 -U mistachkin -Z 3270bf7f35234fd560a264ffe40cbf35 +P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 +R 88b26813bcaf999746e4cf55503b4797 +U drh +Z fe6d8478e3f31cb2dfbec6ce858d4ad8 diff --git a/manifest.uuid b/manifest.uuid index 63b3b0cf2b..12179a84a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 \ No newline at end of file +ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5a3d44f9ee..95bbdb157e 100644 --- a/src/where.c +++ b/src/where.c @@ -1885,16 +1885,15 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y -** (2) X is a proper subset of Y -** (3) X skips at least as many columns as Y -** -** By "proper subset" we mean that X uses fewer WHERE clause terms -** than Y and that every WHERE clause term used by X is also used -** by Y. +** (2) X users fewer WHERE clause terms than Y +** (3) Every WHERE clause term used by X is also used by Y +** (4) X skips at least as many columns as Y +** (5) If X is a covering index, than Y is too ** +** Conditions (2) and (3) mean that X is a "proper subset" of Y. ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. The third rule +** relationship is inverted and needs to be adjusted. Constraint (4) ** was added because if X uses skip-scan less than Y it still might ** deserve a lower cost even if it is a proper subset of Y. */ @@ -1918,6 +1917,10 @@ static int whereLoopCheaperProperSubset( } if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ } + if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 + && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ + return 0; /* Constraint (5) */ + } return 1; /* All conditions meet */ } diff --git a/test/analyze9.test b/test/analyze9.test index 918577bd19..7c69806f89 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1052,8 +1052,11 @@ do_execsql_test 23.0 { do_eqp_test 23.1 { SELECT * FROM t4 WHERE (e=1 AND b='xyz' AND c='zyx' AND a<'AEA') AND f<300 + -- Formerly used index i41. But i41 is not a covering index whereas + -- the PRIMARY KEY is a covering index, and so as of 2017-10-15, the + -- PRIMARY KEY is preferred. } { - 0 0 0 {SEARCH TABLE t4 USING INDEX i41 (e=? AND c=? AND b=? AND a Date: Mon, 16 Oct 2017 11:50:12 +0000 Subject: [PATCH 020/156] Improved comments, including a typo fix, on the whereLoopCheaperProperSubset() function. FossilOrigin-Name: 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b77b1484ce..ab44e57d39 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot. -D 2017-10-15T22:16:25.229 +C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction. +D 2017-10-16T11:50:12.951 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 -F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763 +F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090 -R 88b26813bcaf999746e4cf55503b4797 +P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d +R 593b5394fda7f5c3a7e7e5312c0a0c37 U drh -Z fe6d8478e3f31cb2dfbec6ce858d4ad8 +Z d6213a58e95c0fa7e25d95c2efa3d8ac diff --git a/manifest.uuid b/manifest.uuid index 12179a84a0..3281f881f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d \ No newline at end of file +700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 95bbdb157e..6021edf47f 100644 --- a/src/where.c +++ b/src/where.c @@ -1885,7 +1885,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y -** (2) X users fewer WHERE clause terms than Y +** (2) X uses fewer WHERE clause terms than Y ** (3) Every WHERE clause term used by X is also used by Y ** (4) X skips at least as many columns as Y ** (5) If X is a covering index, than Y is too @@ -1895,7 +1895,9 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** to have a lower cost. This routine returns TRUE when that cost ** relationship is inverted and needs to be adjusted. Constraint (4) ** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. +** deserve a lower cost even if it is a proper subset of Y. Constraint (5) +** was added because a covering index probably deserves to have a lower cost +** than a non-covering index even if it is a proper subset. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ From f30d345241ac0d260c5a8034b1366c5bee596ee3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Oct 2017 13:44:46 +0000 Subject: [PATCH 021/156] Fix the ".check" command of the command-line shell to return 1 (error) rather than 2 (exit) on a test failure. FossilOrigin-Name: e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab44e57d39..de1b01a07e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction. -D 2017-10-16T11:50:12.951 +C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. +D 2017-10-17T13:44:46.075 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -462,7 +462,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 -F src/shell.c.in 5446de0a90c15d713bbdb5827cf57ec30d1c3497097f39ec2c2e874dcca34ca3 +F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d -R 593b5394fda7f5c3a7e7e5312c0a0c37 +P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f +R 7b11fa4b75f3d72b968740b4ad577de5 U drh -Z d6213a58e95c0fa7e25d95c2efa3d8ac +Z eb279a84da6335af377cd536eb14b148 diff --git a/manifest.uuid b/manifest.uuid index 3281f881f9..cd80d78996 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f \ No newline at end of file +e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b62c055076..8f5ed59e93 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4248,7 +4248,7 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(stderr, "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", p->zTestcase, azArg[1], zRes); - rc = 2; + rc = 1; }else{ utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); p->nCheck++; From f7fc4c22f45adcf20fa88febad0a0e046a04f9ae Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 17 Oct 2017 18:33:22 +0000 Subject: [PATCH 022/156] The source code files generated for the 'sqlite3-all.c' target should not have Windows line-endings. FossilOrigin-Name: 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/split-sqlite3c.tcl | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de1b01a07e..d6e49a102d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. -D 2017-10-17T13:44:46.075 +C The\ssource\scode\sfiles\sgenerated\sfor\sthe\s'sqlite3-all.c'\starget\sshould\snot\shave\sWindows\sline-endings. +D 2017-10-17T18:33:22.646 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1630,7 +1630,7 @@ F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd -F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c +F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f -R 7b11fa4b75f3d72b968740b4ad577de5 -U drh -Z eb279a84da6335af377cd536eb14b148 +P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d +R d1f05de03c82774074759a8848af421f +U mistachkin +Z cf3e5ebd3225c433ae20a29b56e94ef7 diff --git a/manifest.uuid b/manifest.uuid index cd80d78996..4c5b7a09c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file +6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 \ No newline at end of file diff --git a/tool/split-sqlite3c.tcl b/tool/split-sqlite3c.tcl index 287b752828..230e3f2549 100644 --- a/tool/split-sqlite3c.tcl +++ b/tool/split-sqlite3c.tcl @@ -15,6 +15,7 @@ set END {^/\*+ End of %s \*+/} set in [open sqlite3.c] set out1 [open sqlite3-all.c w] +fconfigure $out1 -translation lf # Copy the header from sqlite3.c into sqlite3-all.c # @@ -48,6 +49,7 @@ proc write_one_file {content} { global filecnt incr filecnt set out [open sqlite3-$filecnt.c w] + fconfigure $out -translation lf puts -nonewline $out $content close $out puts $::out1 "#include \"sqlite3-$filecnt.c\"" From af124faf3e6e6284161b84e68f7e13235bd6afb4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 18 Oct 2017 14:29:36 +0000 Subject: [PATCH 023/156] Avoid using snprintf as a member variable of a struct in sqlite3ext.h. This causes problems on OSX. FossilOrigin-Name: b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/sqlite3ext.h | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index de1b01a07e..365c4e7bb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure. -D 2017-10-17T13:44:46.075 +C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX. +D 2017-10-18T14:29:36.881 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -465,7 +465,7 @@ F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 +F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 @@ -1664,7 +1664,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f -R 7b11fa4b75f3d72b968740b4ad577de5 -U drh -Z eb279a84da6335af377cd536eb14b148 +P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d +R af758af6d9338e0da5d96870c08db747 +T *branch * sqlite3ext-fix +T *sym-sqlite3ext-fix * +T -sym-trunk * +U dan +Z d17bbc38b957a75c44d151990964eb3a diff --git a/manifest.uuid b/manifest.uuid index cd80d78996..73db2c4c9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d \ No newline at end of file +b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 4aade278bd..d1d2c574ae 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -134,7 +134,7 @@ struct sqlite3_api_routines { int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*snprintf)(int,char*,const char*,...); + char * (*xsnprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, char const**,char const**,int*,int*,int*); @@ -418,7 +418,7 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_rollback_hook sqlite3_api->rollback_hook #define sqlite3_set_authorizer sqlite3_api->set_authorizer #define sqlite3_set_auxdata sqlite3_api->set_auxdata -#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_snprintf sqlite3_api->xsnprintf #define sqlite3_step sqlite3_api->step #define sqlite3_table_column_metadata sqlite3_api->table_column_metadata #define sqlite3_thread_cleanup sqlite3_api->thread_cleanup From f2f72a0f9fe7125285f233011bb10257fa5a6a88 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 19 Oct 2017 15:17:38 +0000 Subject: [PATCH 024/156] Take extra care to avoid an OOB read caused by a corrupt b-tree page. FossilOrigin-Name: 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b --- manifest | 18 +++---- manifest.uuid | 2 +- src/btree.c | 13 +++-- test/corruptK.test | 109 +++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 2 +- 5 files changed, 129 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 75ecef5321..5c8ae4d13c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX.\sSimilar\sto\sfix\s[a1fd3aa8]. -D 2017-10-18T15:02:01.361 +C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage. +D 2017-10-19T15:17:38.752 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -403,7 +403,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 8565b061a6a6fad850230c73d6a7a8ffb88f3370e3352a8689a9a672160c5cc5 +F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673 @@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe +F test/corruptK.test 251ef631d095e882d455d2183961fa9ba879b4156e18e96c5d2b84aa7ef5f7a9 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6 +F test/permutations.test 490e3333b9b1aefb7ebc6e9ab2ae0e382b7dd8713ccc4a2786b0f75467c2ab6b F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1664,8 +1664,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c -R bae812ccbd328c244f2fa888e3ba625e -T +closed b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c +P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 +R 14641e8822b464d39bf472f899e19240 +T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a U dan -Z 4879d15d8231001302e10b2e0a3c2891 +Z 576e55698c218b4128bbad5de39dc1b0 diff --git a/manifest.uuid b/manifest.uuid index 2f3fcd4b40..26a2375f60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 \ No newline at end of file +04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 89222197bc..ddcb6cfd35 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4824,7 +4824,7 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ - u32 amt; + int amt; assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -4833,9 +4833,14 @@ static const void *fetchPayload( assert( pCur->info.nSize>0 ); assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); - amt = (int)(pCur->pPage->aDataEnd - pCur->info.pPayload); - if( pCur->info.nLocalinfo.nLocal; - *pAmt = amt; + amt = pCur->info.nLocal; + if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){ + /* There is too little space on the page for the expected amount + ** of local content. Database must be corrupt. */ + assert( CORRUPT_DB ); + amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload)); + } + *pAmt = (u32)amt; return (void*)pCur->info.pPayload; } diff --git a/test/corruptK.test b/test/corruptK.test index b20c2d8bf0..b921176974 100644 --- a/test/corruptK.test +++ b/test/corruptK.test @@ -107,6 +107,115 @@ do_catchsql_test 2.3 { INSERT INTO t1 VALUES(randomblob(900)); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- + +proc hex2blob {hex} { + # Split on newlines: + set bytes [list] + foreach l [split $hex "\n"] { + if {[string is space $l]} continue + set L [list] + foreach b [split $l] { + if {[string is xdigit $b] && [string length $b]==2} { + lappend L [expr "0x$b"] + } + } + if {[llength $L]!=16} { + error "Badly formed hex (1)" + } + set bytes [concat $bytes $L] + } + + binary format c* $bytes +} + +reset_db +db func hex2blob hex2blob + +do_execsql_test 3.1 { + PRAGMA page_size=1024; + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(a, b, c); + CREATE TABLE t3(a, b, c); + CREATE TABLE t4(a, b, c); + CREATE TABLE t5(a, b, c); +} + +do_execsql_test 3.2 { + UPDATE sqlite_dbpage SET data = hex2blob(' + 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @ ..>..... + 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................ + 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8.. + 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>. + 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-.............. + 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................ + 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0... + 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................ + 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........ + 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................ + 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p. + 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h.......... + 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................ + 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................ + 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p........... + 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese + 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE + 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor + 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s . $ + 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal + 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text) + ') WHERE pgno=1 +} + +db close +sqlite3 db test.db + +do_catchsql_test 3.3 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} diff --git a/test/permutations.test b/test/permutations.test index 1e04265668..76fb72ace5 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -194,7 +194,7 @@ test_suite "valgrind" -prefix "" -description { } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* wal.test \ shell*.test crash8.test atof1.test selectG.test \ - tkt-fc62af4523.test numindex1.test + tkt-fc62af4523.test numindex1.test corruptK.test ] -initialize { set ::G(valgrind) 1 } -shutdown { From 316ab93e2c92526d0fd2b42f53b75be32e6a0428 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Oct 2017 13:29:26 +0000 Subject: [PATCH 025/156] Remove unnecessary "#if SQLITE_MAX_COLUMN". SQLITE_MAX_COLUMN is always defined. FossilOrigin-Name: 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a --- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/build.c | 2 -- src/resolve.c | 4 ---- src/select.c | 2 -- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5c8ae4d13c..cd878b65fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage. -D 2017-10-19T15:17:38.752 +C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined. +D 2017-10-21T13:29:26.479 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -406,7 +406,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673 +F src/build.c e24b61144f9c9b15c4aa05954514190361061da721e56dcd1af6e0e945ee5909 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -459,9 +459,9 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 +F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18 +F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1664,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 -R 14641e8822b464d39bf472f899e19240 -T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a -U dan -Z 576e55698c218b4128bbad5de39dc1b0 +P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b +R 3fe93e66991cd1a89b7cde18365d4335 +U drh +Z 82fbf688aff072892ed2c0d525c25911 diff --git a/manifest.uuid b/manifest.uuid index 26a2375f60..14db0bd9b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b \ No newline at end of file +6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a \ No newline at end of file diff --git a/src/build.c b/src/build.c index cb8c374d1a..a9428104a3 100644 --- a/src/build.c +++ b/src/build.c @@ -1063,12 +1063,10 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; -#if SQLITE_MAX_COLUMN if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); return; } -#endif z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; memcpy(z, pName->z, pName->n); diff --git a/src/resolve.c b/src/resolve.c index 78f37512a2..4a1e8284c8 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -959,12 +959,10 @@ static int resolveCompoundOrderBy( pOrderBy = pSelect->pOrderBy; if( pOrderBy==0 ) return 0; db = pParse->db; -#if SQLITE_MAX_COLUMN if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); return 1; } -#endif for(i=0; inExpr; i++){ pOrderBy->a[i].done = 0; } @@ -1056,12 +1054,10 @@ int sqlite3ResolveOrderGroupBy( struct ExprList_item *pItem; if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; -#if SQLITE_MAX_COLUMN if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); return 1; } -#endif pEList = pSelect->pEList; assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ diff --git a/src/select.c b/src/select.c index 6f524fa2e1..9639cfde1a 100644 --- a/src/select.c +++ b/src/select.c @@ -4596,12 +4596,10 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ExprListDelete(db, pEList); p->pEList = pNew; } -#if SQLITE_MAX_COLUMN if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns in result set"); return WRC_Abort; } -#endif return WRC_Continue; } From 6fe3733ba97193227a395c7796bb9ca0d6f82134 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Oct 2017 14:17:31 +0000 Subject: [PATCH 026/156] Catch and avoid a 16-bit integer overflow on the number of columns in a common table expression. This fixes a problem found by OSS-Fuzz. The test case is in TH3. FossilOrigin-Name: 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cd878b65fe..da985ae44f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined. -D 2017-10-21T13:29:26.479 +C Catch\sand\savoid\sa\s16-bit\sinteger\soverflow\son\sthe\snumber\sof\scolumns\sin\sa\ncommon\stable\sexpression.\s\sThis\sfixes\sa\sproblem\sfound\sby\sOSS-Fuzz.\s\sThe\ntest\scase\sis\sin\sTH3. +D 2017-10-21T14:17:31.555 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -461,7 +461,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367 +F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b -R 3fe93e66991cd1a89b7cde18365d4335 +P 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a +R 201775ba1e14aa2d5f8d0e57defde230 U drh -Z 82fbf688aff072892ed2c0d525c25911 +Z bad3386d6737a5472f5b033811b6e996 diff --git a/manifest.uuid b/manifest.uuid index 14db0bd9b2..60279d9206 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a \ No newline at end of file +6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9639cfde1a..0e2328120b 100644 --- a/src/select.c +++ b/src/select.c @@ -1689,6 +1689,7 @@ int sqlite3ColumnsFromExprList( nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); testcase( aCol==0 ); + if( nCol>32767 ) nCol = 32767; }else{ nCol = 0; aCol = 0; From 9499b4a6143b9464fa27e25dddd6a5301ab352de Mon Sep 17 00:00:00 2001 From: dan Date: Sun, 22 Oct 2017 07:57:29 +0000 Subject: [PATCH 027/156] Avoid running tests that use sqlite_dbpage with SQLITE_OMIT_VIRTUAL_TABLE builds. FossilOrigin-Name: 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528 --- manifest | 14 +-- manifest.uuid | 2 +- test/corruptK.test | 210 +++++++++++++++++++++++---------------------- 3 files changed, 115 insertions(+), 111 deletions(-) diff --git a/manifest b/manifest index da985ae44f..07fa1c88b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Catch\sand\savoid\sa\s16-bit\sinteger\soverflow\son\sthe\snumber\sof\scolumns\sin\sa\ncommon\stable\sexpression.\s\sThis\sfixes\sa\sproblem\sfound\sby\sOSS-Fuzz.\s\sThe\ntest\scase\sis\sin\sTH3. -D 2017-10-21T14:17:31.555 +C Avoid\srunning\stests\sthat\suse\ssqlite_dbpage\swith\sSQLITE_OMIT_VIRTUAL_TABLE\nbuilds. +D 2017-10-22T07:57:29.038 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test 251ef631d095e882d455d2183961fa9ba879b4156e18e96c5d2b84aa7ef5f7a9 +F test/corruptK.test d93f5797bf0555a47dcc81bc9727cb72fa7c8ac348c25f42937573cd29bb3850 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a -R 201775ba1e14aa2d5f8d0e57defde230 -U drh -Z bad3386d6737a5472f5b033811b6e996 +P 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c +R 2ad0f857ce1c547a47a99955708ad02e +U dan +Z 31291a1d9ccb37db331d41df009b953d diff --git a/manifest.uuid b/manifest.uuid index 60279d9206..321fef9696 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c \ No newline at end of file +7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528 \ No newline at end of file diff --git a/test/corruptK.test b/test/corruptK.test index b921176974..05465a9b5f 100644 --- a/test/corruptK.test +++ b/test/corruptK.test @@ -109,113 +109,117 @@ do_catchsql_test 2.3 { #------------------------------------------------------------------------- -proc hex2blob {hex} { - # Split on newlines: - set bytes [list] - foreach l [split $hex "\n"] { - if {[string is space $l]} continue - set L [list] - foreach b [split $l] { - if {[string is xdigit $b] && [string length $b]==2} { - lappend L [expr "0x$b"] +ifcapable vtab { + + proc hex2blob {hex} { + # Split on newlines: + set bytes [list] + foreach l [split $hex "\n"] { + if {[string is space $l]} continue + set L [list] + foreach b [split $l] { + if {[string is xdigit $b] && [string length $b]==2} { + lappend L [expr "0x$b"] + } } + if {[llength $L]!=16} { + error "Badly formed hex (1)" + } + set bytes [concat $bytes $L] } - if {[llength $L]!=16} { - error "Badly formed hex (1)" - } - set bytes [concat $bytes $L] + + binary format c* $bytes } + + reset_db + db func hex2blob hex2blob + + do_execsql_test 3.1 { + PRAGMA page_size=1024; + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(a, b, c); + CREATE TABLE t3(a, b, c); + CREATE TABLE t4(a, b, c); + CREATE TABLE t5(a, b, c); + } + + do_execsql_test 3.2 { + UPDATE sqlite_dbpage SET data = hex2blob(' + 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @ ..>..... + 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................ + 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8.. + 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>. + 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-.............. + 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................ + 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0... + 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................ + 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........ + 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................ + 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p. + 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h.......... + 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................ + 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................ + 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p........... + 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese + 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE + 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor + 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s . $ + 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal + 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text) + ') WHERE pgno=1 + } + + db close + sqlite3 db test.db + + do_catchsql_test 3.3 { + PRAGMA integrity_check; + } {1 {database disk image is malformed}} - binary format c* $bytes -} - -reset_db -db func hex2blob hex2blob - -do_execsql_test 3.1 { - PRAGMA page_size=1024; - CREATE TABLE t1(a, b, c); - CREATE TABLE t2(a, b, c); - CREATE TABLE t3(a, b, c); - CREATE TABLE t4(a, b, c); - CREATE TABLE t5(a, b, c); -} - -do_execsql_test 3.2 { - UPDATE sqlite_dbpage SET data = hex2blob(' - 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. - 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @ ..>..... - 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ - 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................ - 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8.. - 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>. - 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-.............. - 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................ - 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0... - 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................ - 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........ - 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................ - 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p. - 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h.......... - 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................ - 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................ - 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p........... - 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ - 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese - 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE - 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor - 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s . $ - 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal - 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text) - ') WHERE pgno=1 -} - -db close -sqlite3 db test.db - -do_catchsql_test 3.3 { - PRAGMA integrity_check; -} {1 {database disk image is malformed}} +} ;# ifcapable vtab From 063cf17a113a79b503d8dc6df7989414b6804a83 Mon Sep 17 00:00:00 2001 From: dan Date: Sun, 22 Oct 2017 08:02:56 +0000 Subject: [PATCH 028/156] Fix test cases in dbpage.test so that they work with SQLITE_DEFAULT_AUTOVACUUM=1 builds. FossilOrigin-Name: 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/dbpage.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 07fa1c88b3..68f0dd8fc9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\srunning\stests\sthat\suse\ssqlite_dbpage\swith\sSQLITE_OMIT_VIRTUAL_TABLE\nbuilds. -D 2017-10-22T07:57:29.038 +C Fix\stest\scases\sin\sdbpage.test\sso\sthat\sthey\swork\swith\nSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. +D 2017-10-22T08:02:56.288 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -713,7 +713,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbpage.test 9cf4dc92a4de67c81e5c32b24e3fbb8c4757e4b642694a219b3090a4f9277a4d +F test/dbpage.test a9e7774416f5f2fb237601628804b339f897112edb9ba92f7675a61e3e9c6244 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c -R 2ad0f857ce1c547a47a99955708ad02e +P 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528 +R 6416cb828bdd2d9e1b57b650566d1c6c U dan -Z 31291a1d9ccb37db331d41df009b953d +Z 9dddfd6ebe074eda07e6c2b88ec3bea0 diff --git a/manifest.uuid b/manifest.uuid index 321fef9696..e9c39dc6ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528 \ No newline at end of file +0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7 \ No newline at end of file diff --git a/test/dbpage.test b/test/dbpage.test index e29d4b33a9..e4389a3b90 100644 --- a/test/dbpage.test +++ b/test/dbpage.test @@ -22,6 +22,7 @@ ifcapable !vtab||!compound { } do_execsql_test 100 { + PRAGMA auto_vacuum=0; PRAGMA page_size=4096; PRAGMA journal_mode=WAL; CREATE TABLE t1(a,b); From f970372698fc57380c8756b3809ed13ca246dcec Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 23 Oct 2017 07:52:07 +0000 Subject: [PATCH 029/156] Fix minor test script problems revealed by releasetest.tcl. FossilOrigin-Name: 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/corruptK.test | 2 ++ test/dbpage.test | 22 +++++++++++++--------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 68f0dd8fc9..994305976a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scases\sin\sdbpage.test\sso\sthat\sthey\swork\swith\nSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. -D 2017-10-22T08:02:56.288 +C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl. +D 2017-10-23T07:52:07.660 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test d93f5797bf0555a47dcc81bc9727cb72fa7c8ac348c25f42937573cd29bb3850 +F test/corruptK.test 91550557849244a9904f4e090052e3f2c1c3f1106840d58b00ffaa3a8c2d3fc0 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -713,7 +713,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbpage.test a9e7774416f5f2fb237601628804b339f897112edb9ba92f7675a61e3e9c6244 +F test/dbpage.test 10b9e91d07b0892444fff4578706648e955b5fb260218298f838da74f0d9d211 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528 -R 6416cb828bdd2d9e1b57b650566d1c6c +P 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7 +R e3bec412c3fcc940b80e7c9735d18bbe U dan -Z 9dddfd6ebe074eda07e6c2b88ec3bea0 +Z a4aaac778f287d83315a30f9e1092784 diff --git a/manifest.uuid b/manifest.uuid index e9c39dc6ef..be55c08755 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7 \ No newline at end of file +15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 \ No newline at end of file diff --git a/test/corruptK.test b/test/corruptK.test index 05465a9b5f..1694db1747 100644 --- a/test/corruptK.test +++ b/test/corruptK.test @@ -110,6 +110,7 @@ do_catchsql_test 2.3 { #------------------------------------------------------------------------- ifcapable vtab { +if {[permutation]!="inmemory_journal"} { proc hex2blob {hex} { # Split on newlines: @@ -219,6 +220,7 @@ ifcapable vtab { PRAGMA integrity_check; } {1 {database disk image is malformed}} +} ;# [permutation]!="inmemory_journal" } ;# ifcapable vtab diff --git a/test/dbpage.test b/test/dbpage.test index e4389a3b90..a8ef8c0800 100644 --- a/test/dbpage.test +++ b/test/dbpage.test @@ -21,15 +21,19 @@ ifcapable !vtab||!compound { return } -do_execsql_test 100 { - PRAGMA auto_vacuum=0; - PRAGMA page_size=4096; - PRAGMA journal_mode=WAL; - CREATE TABLE t1(a,b); - WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) - INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c; - PRAGMA integrity_check; -} {wal ok} +do_test 100 { + execsql { + PRAGMA auto_vacuum=0; + PRAGMA page_size=4096; + PRAGMA journal_mode=WAL; + } + execsql { + CREATE TABLE t1(a,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c; + PRAGMA integrity_check; + } +} {ok} do_execsql_test 110 { SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno; } {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0D00000016'} From b5ef728db66284c47861ea66a7e906a9a4b33cb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 23 Oct 2017 11:48:00 +0000 Subject: [PATCH 030/156] In lsm (not SQLite), avoid opening the *-shm file before the database file is locked. Otherwise, there is a race condition whereby a connection in another process process may unlink() it while it is still in use. FossilOrigin-Name: c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad --- ext/lsm1/lsm_shared.c | 10 ++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c index 83e44b4705..c4bb9d6a23 100644 --- a/ext/lsm1/lsm_shared.c +++ b/ext/lsm1/lsm_shared.c @@ -340,9 +340,6 @@ static int doDbConnect(lsm_db *pDb){ /* Obtain a pointer to the shared-memory header */ assert( pDb->pShmhdr==0 ); assert( pDb->bReadonly==0 ); - rc = lsmShmCacheChunks(pDb, 1); - if( rc!=LSM_OK ) return rc; - pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ @@ -353,10 +350,11 @@ static int doDbConnect(lsm_db *pDb){ nUs = nUs * 2; if( nUs>nUsMax ) nUs = nUsMax; } - if( rc!=LSM_OK ){ - pDb->pShmhdr = 0; - return rc; + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(pDb, 1); } + if( rc!=LSM_OK ) return rc; + pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first ** and only connection to the database. In this case initialize the diff --git a/manifest b/manifest index 994305976a..3bd79dea76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl. -D 2017-10-23T07:52:07.660 +C In\slsm\s(not\sSQLite),\savoid\sopening\sthe\s*-shm\sfile\sbefore\sthe\sdatabase\sfile\sis\nlocked.\sOtherwise,\sthere\sis\sa\srace\scondition\swhereby\sa\sconnection\sin\sanother\nprocess\sprocess\smay\sunlink()\sit\swhile\sit\sis\sstill\sin\suse. +D 2017-10-23T11:48:00.248 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -245,7 +245,7 @@ F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709 F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea -F ext/lsm1/lsm_shared.c 5bc37768e558492f60d7196735ddd54843cd239bd66c1af6eb205a6348ca5e46 +F ext/lsm1/lsm_shared.c 1a76b7a5e89a003c24d58f1fb295c4203b48ef6acba9a194ac6003ade09fcd47 F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7 -R e3bec412c3fcc940b80e7c9735d18bbe +P 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 +R 788a72b4e4bc49507efed4d645930eed U dan -Z a4aaac778f287d83315a30f9e1092784 +Z 97cc68bc0ee206aadd31e81854de499f diff --git a/manifest.uuid b/manifest.uuid index be55c08755..abddaa098d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 \ No newline at end of file +c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad \ No newline at end of file From 21cd29ab15630e29dc75dadbd8bb8d1e8fffbfc7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 23 Oct 2017 16:03:54 +0000 Subject: [PATCH 031/156] Avoid passing a NULL value as the second argument to memcpy(). FossilOrigin-Name: 58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- src/vdbe.c | 11 +++-------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 5ff2166ddf..5af1c47e9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl. -D 2017-10-23T07:53:00.457 +C Avoid\spassing\sa\sNULL\svalue\sas\sthe\ssecond\sargument\sto\smemcpy(). +D 2017-10-23T16:03:54.957 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -414,7 +414,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 -F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a +F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6 @@ -530,7 +530,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373 +F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9474f061d634721ca462bbad375270b8b39dea44011df3e1aac72021e8bf70fc 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 -R fca4c2998d860b91137d3cb7520f7137 +P c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831 +R 676a97200c74a615c9238afe630ac524 U dan -Z 456ef7edc59a34c970d90601a596e58f +Z 81b9a8d334381cc6ba3e8852cc37f621 diff --git a/manifest.uuid b/manifest.uuid index 97cae8c731..7a241c2c14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831 \ No newline at end of file +58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0b922cd0fd..31e9a93865 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2678,7 +2678,7 @@ int sqlite3CodeSubselect( /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempReg(pParse); - if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + if( isRowid ) sqlite3VdbeAddOp4(v, OP_Blob, 0, r2, 0, "", P4_STATIC); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; int iValToIns; diff --git a/src/vdbe.c b/src/vdbe.c index 9687170bec..cfe18a9d1e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4435,14 +4435,9 @@ case OP_InsertInt: { if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; - if( pData->flags & MEM_Null ){ - x.pData = 0; - x.nData = 0; - }else{ - assert( pData->flags & (MEM_Blob|MEM_Str) ); - x.pData = pData->z; - x.nData = pData->n; - } + assert( pData->flags & (MEM_Blob|MEM_Str) ); + x.pData = pData->z; + x.nData = pData->n; seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); if( pData->flags & MEM_Zero ){ x.nZero = pData->u.nZero; From efeaec365f44f69576158411bd7f7a27304b7e64 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 23 Oct 2017 16:34:07 +0000 Subject: [PATCH 032/156] Updates to test procedures so that they all work with encryption. No changes to the core SQLite code. FossilOrigin-Name: 0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/test_config.c | 6 ++++++ test/pragma.test | 43 ++++++++++++++++++++++--------------------- test/schema6.test | 1 + test/swarmvtab.test | 2 +- test/swarmvtab2.test | 1 + 7 files changed, 43 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 5af1c47e9c..4bf6a1d006 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\spassing\sa\sNULL\svalue\sas\sthe\ssecond\sargument\sto\smemcpy(). -D 2017-10-23T16:03:54.957 +C Updates\sto\stest\sprocedures\sso\sthat\sthey\sall\swork\swith\sencryption.\s\sNo\schanges\nto\sthe\score\sSQLite\scode. +D 2017-10-23T16:34:07.586 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -486,7 +486,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c abf6fc1fe9d041b699578c42e3db81f8831c4f5b804f1927958102ee8f2b773e +F src/test_config.c 3000f00b9b47b149d842059904c3fcab5f3871fb6aee7d7cc5756f0c64779ae3 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 @@ -1106,7 +1106,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test 490e3333b9b1aefb7ebc6e9ab2ae0e382b7dd8713ccc4a2786b0f75467c2ab6b -F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15 +F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b335c @@ -1163,7 +1163,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e -F test/schema6.test 5b21bbdd405bc93b3e6af5e6ece64d230e35f65cc4035e5c2b89fc8a090d7270 +F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce1f9b F test/securedel.test 5f997cb6bd38727b81e0985f53ec386c99db6441b2b9e6357240649d29017239 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7 @@ -1249,8 +1249,8 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 -F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17 -F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d +F test/swarmvtab.test c2279311b44de032f86a8295a9b06818d864856f9428b4c99eee91a0d419cf25 +F test/swarmvtab2.test 9a3a68a1e58d00f4ed6c68d12d52f2df971b9e22a80a41f6f8c1409abba8e5b4 F test/swarmvtabfault.test 73563eefe3073c6fb3bb14475fb4ef5d4f2e3a67a02947ee0ca08980ea3dd7fe F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831 -R 676a97200c74a615c9238afe630ac524 -U dan -Z 81b9a8d334381cc6ba3e8852cc37f621 +P 58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577 +R a447261df1989a937247d3824cdf45d6 +U drh +Z c50c87663ba16acea2ad67bd15f69a8a diff --git a/manifest.uuid b/manifest.uuid index 7a241c2c14..3a2fc73c3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577 \ No newline at end of file +0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 9ff84dcec0..193f95edf9 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -214,6 +214,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "json1", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_HAS_CODEC + Tcl_SetVar2(interp, "sqlite_options", "has_codec", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "has_codec", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS Tcl_SetVar2(interp, "sqlite_options", "like_match_blobs", "0", TCL_GLOBAL_ONLY); #else diff --git a/test/pragma.test b/test/pragma.test index e4c79111ba..90cd7e2dfe 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1934,26 +1934,27 @@ do_test 23.5 { } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE} db2 close -reset_db -do_execsql_test 24.0 { - PRAGMA page_size = 1024; - CREATE TABLE t1(a, b, c); - CREATE INDEX i1 ON t1(b); - INSERT INTO t1 VALUES('a', 'b', 'c'); - PRAGMA integrity_check; -} {ok} - -set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}] -db close -hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263 - -sqlite3 db test.db -do_catchsql_test 24.1 { - SELECT * FROM t1; -} {1 {database disk image is malformed}} -do_catchsql_test 24.2 { - PRAGMA integrity_check; -} {0 {{database disk image is malformed}}} - +ifcapable !has_codec { + reset_db + do_execsql_test 24.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES('a', 'b', 'c'); + PRAGMA integrity_check; + } {ok} + + set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}] + db close + hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263 + + sqlite3 db test.db + do_catchsql_test 24.1 { + SELECT * FROM t1; + } {1 {database disk image is malformed}} + do_catchsql_test 24.2 { + PRAGMA integrity_check; + } {0 {{database disk image is malformed}}} +} database_never_corrupt finish_test diff --git a/test/schema6.test b/test/schema6.test index 7de04d51c2..0eb8a1c434 100644 --- a/test/schema6.test +++ b/test/schema6.test @@ -18,6 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix schema6 +do_not_use_codec # Command: check_same_database_content TESTNAME SQL1 SQL2 SQL3 ... # diff --git a/test/swarmvtab.test b/test/swarmvtab.test index c8128ef181..4cdcf29ca8 100644 --- a/test/swarmvtab.test +++ b/test/swarmvtab.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix swarmvtab +do_not_use_codec ifcapable !vtab { finish_test @@ -243,4 +244,3 @@ do_catchsql_test 3.3.2 { SELECT * FROM xyz } {1 {fetch_db error!}} finish_test - diff --git a/test/swarmvtab2.test b/test/swarmvtab2.test index 2b37fd2ac8..cf1bdd0af9 100644 --- a/test/swarmvtab2.test +++ b/test/swarmvtab2.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix swarmvtab +do_not_use_codec ifcapable !vtab { finish_test From 15dd780a9eb97f31368410299549f5ca0b70491b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Oct 2017 17:36:52 +0000 Subject: [PATCH 033/156] Cleanup a test database file prior to using it. FossilOrigin-Name: e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/swarmvtabfault.test | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3bd79dea76..336384dfde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\slsm\s(not\sSQLite),\savoid\sopening\sthe\s*-shm\sfile\sbefore\sthe\sdatabase\sfile\sis\nlocked.\sOtherwise,\sthere\sis\sa\srace\scondition\swhereby\sa\sconnection\sin\sanother\nprocess\sprocess\smay\sunlink()\sit\swhile\sit\sis\sstill\sin\suse. -D 2017-10-23T11:48:00.248 +C Cleanup\sa\stest\sdatabase\sfile\sprior\sto\susing\sit. +D 2017-10-23T17:36:52.561 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1251,7 +1251,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17 F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d -F test/swarmvtabfault.test 73563eefe3073c6fb3bb14475fb4ef5d4f2e3a67a02947ee0ca08980ea3dd7fe +F test/swarmvtabfault.test f8ae84cd44454555386c393892280f85f5ab35dcd0797641830fd7ebbfcc9d15 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1 -R 788a72b4e4bc49507efed4d645930eed -U dan -Z 97cc68bc0ee206aadd31e81854de499f +P c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad +R 8a4b9bbdc7dfd424f00b7f6b2d5817a7 +U mistachkin +Z b3872513ac1c114df2a01eba444b5f49 diff --git a/manifest.uuid b/manifest.uuid index abddaa098d..3ab3d6091f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad \ No newline at end of file +e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b \ No newline at end of file diff --git a/test/swarmvtabfault.test b/test/swarmvtabfault.test index 8c913c1066..dfdd16705b 100644 --- a/test/swarmvtabfault.test +++ b/test/swarmvtabfault.test @@ -29,6 +29,8 @@ proc fetch_db {file} { } forcedelete test.db1 +forcedelete test.db2 + do_execsql_test 1.0 { ATTACH 'test.db1' AS aux; CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b); From 0295774ca0d2d33c7644688be3624433e29ab84b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Oct 2017 20:17:19 +0000 Subject: [PATCH 034/156] In the 'swarmvtabfault' test module, make sure to close the database handle prior to raising an error from the 'not found' callback script. FossilOrigin-Name: f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/swarmvtabfault.test | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 336384dfde..2cb9e12473 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanup\sa\stest\sdatabase\sfile\sprior\sto\susing\sit. -D 2017-10-23T17:36:52.561 +C In\sthe\s'swarmvtabfault'\stest\smodule,\smake\ssure\sto\sclose\sthe\sdatabase\shandle\sprior\sto\sraising\san\serror\sfrom\sthe\s'not\sfound'\scallback\sscript. +D 2017-10-23T20:17:19.102 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1251,7 +1251,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17 F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d -F test/swarmvtabfault.test f8ae84cd44454555386c393892280f85f5ab35dcd0797641830fd7ebbfcc9d15 +F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad -R 8a4b9bbdc7dfd424f00b7f6b2d5817a7 +P e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b +R c767dcd50a85f97407be83eb8fcdc039 U mistachkin -Z b3872513ac1c114df2a01eba444b5f49 +Z 6d76c482979023bebbeef80b070aef99 diff --git a/manifest.uuid b/manifest.uuid index 3ab3d6091f..0c9cd50350 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b \ No newline at end of file +f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b \ No newline at end of file diff --git a/test/swarmvtabfault.test b/test/swarmvtabfault.test index dfdd16705b..083d80df2e 100644 --- a/test/swarmvtabfault.test +++ b/test/swarmvtabfault.test @@ -24,8 +24,11 @@ ifcapable !vtab { proc fetch_db {file} { forcedelete $file sqlite3 dbX $file - dbX eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b) } + set rc [catch { + dbX eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b) } + } res] dbX close + if {$rc!=0} {error $res} } forcedelete test.db1 From f7902a543eceb3dbae5d9fea51bf83ed16a8d4c6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 23 Oct 2017 21:24:38 +0000 Subject: [PATCH 035/156] On the amalgamation package, enable the sqlite_dbpage virtual table so that the .dbinfo command works in the CLI. FossilOrigin-Name: 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6 --- autoconf/Makefile.am | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index e8211596d9..8c046f822d 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h EXTRA_sqlite3_SOURCES = sqlite3.c sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB include_HEADERS = sqlite3.h sqlite3ext.h diff --git a/manifest b/manifest index 2399bd7eea..ce4682de02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sto\sswarmvtabfault.test\sand\sthe\slsm\sextension\sinto\sthis\nbranch.\sNo\schanges\sto\score\sSQLite\scode. -D 2017-10-23T20:20:42.832 +C On\sthe\samalgamation\spackage,\senable\sthe\ssqlite_dbpage\svirtual\stable\sso\sthat\nthe\s.dbinfo\scommand\sworks\sin\sthe\sCLI. +D 2017-10-23T21:24:38.787 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -10,7 +10,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 +F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39 F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b -R ddc3003da6d6bb8e92c74828b5fa4343 -U dan -Z d8722be9be441804446b8c51f09c2f27 +P f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77 +R f52ceae69cf9eb99d83d95eeda09f025 +U drh +Z 919379e1b338add2dc31d98f7ca8060a diff --git a/manifest.uuid b/manifest.uuid index 1fd8f6b26b..5ff9088210 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77 \ No newline at end of file +54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6 \ No newline at end of file From 952523f618d75ffe9e085b65cab63f53684c464a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 24 Oct 2017 17:28:25 +0000 Subject: [PATCH 036/156] Fix a problem in the incr-blob code causing a new cursor to be opened for every sqlite3_blob_reopen() call. FossilOrigin-Name: 41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeblob.c | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ce4682de02..3ca17dd781 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sthe\samalgamation\spackage,\senable\sthe\ssqlite_dbpage\svirtual\stable\sso\sthat\nthe\s.dbinfo\scommand\sworks\sin\sthe\sCLI. -D 2017-10-23T21:24:38.787 +C Fix\sa\sproblem\sin\sthe\sincr-blob\scode\scausing\sa\snew\scursor\sto\sbe\sopened\sfor\nevery\ssqlite3_blob_reopen()\scall. +D 2017-10-24T17:28:25.843 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -535,7 +535,7 @@ F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 F src/vdbeaux.c c423065d50cee24bc8cba57764f5e9869a1bb920c50907f5dd363ebd7c5aef82 -F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286 +F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77 -R f52ceae69cf9eb99d83d95eeda09f025 -U drh -Z 919379e1b338add2dc31d98f7ca8060a +P 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6 +R ba55e91a353da744095752f95418b3a4 +U dan +Z 67fa09a204d6f78af71556879bc0e0f6 diff --git a/manifest.uuid b/manifest.uuid index 5ff9088210..7c05a49d6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6 \ No newline at end of file +41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c \ No newline at end of file diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 16a2c0f36f..4279792696 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -63,11 +63,12 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ v->aMem[1].u.i = iRow; /* If the statement has been run before (and is paused at the OP_ResultRow) - ** then back it up to the point where it does the OP_SeekRowid. This could + ** then back it up to the point where it does the OP_NotExists. This could ** have been down with an extra OP_Goto, but simply setting the program ** counter is faster. */ - if( v->pc>3 ){ - v->pc = 3; + if( v->pc>4 ){ + v->pc = 4; + assert( v->aOp[v->pc].opcode==OP_NotExists ); rc = sqlite3VdbeExec(v); }else{ rc = sqlite3_step(p->pStmt); From 39f2b329f6513b6901afb461b57674a109e934ff Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 24 Oct 2017 18:55:49 +0000 Subject: [PATCH 037/156] Version 3.21.0 FossilOrigin-Name: 1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827 --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 3ca17dd781..1a60fff9b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sincr-blob\scode\scausing\sa\snew\scursor\sto\sbe\sopened\sfor\nevery\ssqlite3_blob_reopen()\scall. -D 2017-10-24T17:28:25.843 +C Version\s3.21.0 +D 2017-10-24T18:55:49.833 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1664,7 +1664,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6 +P 41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c R ba55e91a353da744095752f95418b3a4 -U dan -Z 67fa09a204d6f78af71556879bc0e0f6 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.21.0 * +U drh +Z c165baaf48a254c276ef91f282f216a1 diff --git a/manifest.uuid b/manifest.uuid index 7c05a49d6e..b772210fa4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c \ No newline at end of file +1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827 \ No newline at end of file From ea78f64dc11da4468621bee30a8b6f01fc1ba25e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 24 Oct 2017 21:17:12 +0000 Subject: [PATCH 038/156] Force the autoconf makefile for MSVC to always use the 'find.exe' included with Windows. FossilOrigin-Name: b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887 --- autoconf/Makefile.msc | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- tool/mkmsvcmin.tcl | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index e0f7ad8615..39831da03e 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -974,7 +974,7 @@ sqlite3.lo: $(SQLITE3C) !IF $(USE_RC)!=0 _HASHCHAR=^# !IF ![echo !IFNDEF VERSION > rcver.vc] && \ - ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \ + ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \ ![echo !ENDIF >> rcver.vc] !INCLUDE rcver.vc !ENDIF diff --git a/manifest b/manifest index 918de05b01..d4ab8e50ee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sof\sthe\sfixes\sfrom\sthe\s3.21.0\srelease. -D 2017-10-24T19:05:26.063 +C Force\sthe\sautoconf\smakefile\sfor\sMSVC\sto\salways\suse\sthe\s'find.exe'\sincluded\swith\sWindows. +D 2017-10-24T21:17:12.950 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39 -F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715 +F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 @@ -1596,7 +1596,7 @@ F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c7 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c -F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4 +F tool/mkmsvcmin.tcl 8baf26690b80d861d0ac341b29880eec6ade39e4f11fe690271ded9cb90563a3 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b 1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827 -R ba55e91a353da744095752f95418b3a4 -U drh -Z c9856165fc8b1b55de6e521677f40ace +P 5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441 +R 54e49932e96b845a01a6eb1a3cc08724 +U mistachkin +Z 580a702e0e85c423f77f3c922262f566 diff --git a/manifest.uuid b/manifest.uuid index 9810bd8910..5f2c913791 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441 \ No newline at end of file +b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887 \ No newline at end of file diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 8d5729865c..1d646ef9f5 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -54,7 +54,7 @@ proc substVars { data } { set blocks(1) [string trimleft [string map [list \\\\ \\] { _HASHCHAR=^# !IF ![echo !IFNDEF VERSION > rcver.vc] && \\ - ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\ + ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\ ![echo !ENDIF >> rcver.vc] !INCLUDE rcver.vc !ENDIF From 98d529b5411816e75a038f10e4666c20a42a7628 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Oct 2017 01:34:24 +0000 Subject: [PATCH 039/156] Add the sqlite_btreeinfo eponymous-only table for introspecting the schema and estimating the sizes of various btrees. FossilOrigin-Name: 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d --- ext/misc/btreeinfo.c | 425 +++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +- manifest.uuid | 2 +- 3 files changed, 433 insertions(+), 7 deletions(-) create mode 100644 ext/misc/btreeinfo.c diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c new file mode 100644 index 0000000000..131c63e835 --- /dev/null +++ b/ext/misc/btreeinfo.c @@ -0,0 +1,425 @@ +/* +** 2017-10-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_btreeinfo" virtual table. +** +** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual +** table that shows information about all btrees in an SQLite database file. +** The schema is like this: +** +** CREATE TABLE sqlite_btreeinfo( +** type TEXT, -- "table" or "index" +** name TEXT, -- Name of table or index for this btree. +** tbl_name TEXT, -- Associated table +** rootpage INT, -- The root page of the btree +** sql TEXT, -- SQL for this btree - from sqlite_master +** hasRowid BOOLEAN, -- True if the btree has a rowid +** nEntry INT, -- Estimated number of enteries +** nPage INT, -- Estimated number of pages +** depth INT, -- Depth of the btree +** szPage INT, -- Size of each page in bytes +** zSchema TEXT HIDDEN -- The schema to which this btree belongs +** ); +** +** The first 5 fields are taken directly from the sqlite_master table. +** Considering only the first 5 fields, the only difference between +** this virtual table and the sqlite_master table is that this virtual +** table omits all entries that have a 0 or NULL rowid - in other words +** it omits triggers and views. +** +** The value added by this table comes in the next 5 fields. +** +** Note that nEntry and nPage are *estimated*. They are computed doing +** a single search from the root to a leaf, counting the number of cells +** at each level, and assuming that unvisited pages have a similar number +** of cells. +** +** The sqlite_dbpage virtual table must be available for this virtual table +** to operate. +** +** USAGE EXAMPLES: +** +** Show the table btrees in a schema order with the tables with the most +** rows occuring first: +** +** SELECT name, nEntry +** FROM sqlite_btreeinfo +** WHERE type='table' +** ORDER BY nEntry DESC, name; +** +** Show the names of all WITHOUT ROWID tables: +** +** SELECT name FROM sqlite_btreeinfo +** WHERE type='table' AND NOT hasRowid; +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Columns available in this virtual table */ +#define BINFO_COLUMN_TYPE 0 +#define BINFO_COLUMN_NAME 1 +#define BINFO_COLUMN_TBL_NAME 2 +#define BINFO_COLUMN_ROOTPAGE 3 +#define BINFO_COLUMN_SQL 4 +#define BINFO_COLUMN_HASROWID 5 +#define BINFO_COLUMN_NENTRY 6 +#define BINFO_COLUMN_NPAGE 7 +#define BINFO_COLUMN_DEPTH 8 +#define BINFO_COLUMN_SZPAGE 9 +#define BINFO_COLUMN_SCHEMA 10 + +/* Forward declarations */ +typedef struct BinfoTable BinfoTable; +typedef struct BinfoCursor BinfoCursor; + +/* A cursor for the sqlite_btreeinfo table */ +struct BinfoCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* Query against sqlite_master */ + int rc; /* Result of previous sqlite_step() call */ + int hasRowid; /* hasRowid value. Negative if unknown. */ + sqlite3_int64 nEntry; /* nEntry value */ + int nPage; /* nPage value */ + int depth; /* depth value */ + int szPage; /* size of a btree page. 0 if unknown */ + char *zSchema; /* Schema being interrogated */ +}; + +/* The sqlite_btreeinfo table */ +struct BinfoTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The databse connection */ +}; + +/* +** Connect to the sqlite_btreeinfo virtual table. +*/ +static int binfoConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + BinfoTable *pTab = 0; + int rc = SQLITE_OK; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(\n" + " type TEXT,\n" + " name TEXT,\n" + " tbl_name TEXT,\n" + " rootpage INT,\n" + " sql TEXT,\n" + " hasRowid BOOLEAN,\n" + " nEntry INT,\n" + " nPage INT,\n" + " depth INT,\n" + " szPage INT,\n" + " zSchema TEXT HIDDEN\n" + ")"); + if( rc==SQLITE_OK ){ + pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable)); + if( pTab==0 ) rc = SQLITE_NOMEM; + } + assert( rc==SQLITE_OK || pTab==0 ); + if( pTab ){ + pTab->db = db; + } + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a btreeinfo virtual table. +*/ +static int binfoDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 Use "main" for the schema +** 1 Schema identified by parameter ?1 +*/ +static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + pIdxInfo->estimatedCost = 10000.0; /* Cost estimate */ + pIdxInfo->estimatedRows = 100; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable + && p->iColumn==BINFO_COLUMN_SCHEMA + && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pIdxInfo->estimatedCost = 1000.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + } + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + BinfoCursor *pCsr; + + pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(BinfoCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a btreeinfo cursor. +*/ +static int binfoClose(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->zSchema); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int binfoNext(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + pCsr->rc = sqlite3_step(pCsr->pStmt); + pCsr->hasRowid = -1; + return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int binfoEof(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + return pCsr->rc!=SQLITE_ROW; +} + +/* Position a cursor back to the beginning. +*/ +static int binfoFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + BinfoTable *pTab = (BinfoTable *)pCursor->pVtab; + char *zSql; + int rc; + + sqlite3_free(pCsr->zSchema); + if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + }else{ + pCsr->zSchema = sqlite3_mprintf("main"); + } + zSql = sqlite3_mprintf( + "SELECT 0, 'table','sqlite_master','sqlite_master',1,NULL " + "UNION ALL " + "SELECT rowid, type, name, tbl_name, rootpage, sql" + " FROM \"%w\".sqlite_master WHERE rootpage>=1", + pCsr->zSchema); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + pCsr->hasRowid = -1; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + rc = binfoNext(pCursor); + } + return rc; +} + +/* Decode big-endian integers */ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +/* Examine the b-tree rooted at pgno and estimate its size. +** Return non-zero if anything goes wrong. +*/ +static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ + sqlite3_int64 nEntry = 1; + int nPage = 1; + unsigned char *aData; + sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; + int pgsz = 0; + int nCell; + int iCell; + + rc = sqlite3_prepare_v2(db, + "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1, + &pStmt, 0); + if( rc ) return rc; + pCsr->depth = 1; + while(1){ + sqlite3_bind_int(pStmt, 1, pgno); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + rc = SQLITE_ERROR; + break; + } + pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0); + aData = (unsigned char*)sqlite3_column_blob(pStmt, 0); + if( aData==0 ){ + rc = SQLITE_NOMEM; + break; + } + if( pgno==1 ){ + aData += 100; + pgsz -= 100; + } + pCsr->hasRowid = aData[0]!=2 && aData[0]!=10; + nCell = get_uint16(aData+3); + nEntry *= (nCell+1); + if( aData[0]==10 || aData[0]==13 ) break; + nPage *= (nCell+1); + if( nCell<=1 ){ + pgno = get_uint32(aData+8); + }else{ + iCell = get_uint16(aData+12+2*(nCell/2)); + if( pgno==1 ) iCell -= 100; + if( iCell<=12 || iCell>=pgsz-4 ){ + rc = SQLITE_CORRUPT; + break; + } + pgno = get_uint32(aData+iCell); + } + pCsr->depth++; + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + pCsr->nPage = nPage; + pCsr->nEntry = nEntry; + if( rc==SQLITE_ROW ) rc = SQLITE_OK; + return rc; +} + +/* Return a column for the sqlite_btreeinfo table */ +static int binfoColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){ + int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1); + sqlite3 *db = sqlite3_context_db_handle(ctx); + int rc = binfoCompute(db, pgno, pCsr); + if( rc ){ + return rc; + } + } + switch( i ){ + case BINFO_COLUMN_NAME: + case BINFO_COLUMN_TYPE: + case BINFO_COLUMN_TBL_NAME: + case BINFO_COLUMN_ROOTPAGE: + case BINFO_COLUMN_SQL: { + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1)); + break; + } + case BINFO_COLUMN_HASROWID: { + sqlite3_result_int(ctx, pCsr->hasRowid); + break; + } + case BINFO_COLUMN_NENTRY: { + sqlite3_result_int64(ctx, pCsr->nEntry); + break; + } + case BINFO_COLUMN_NPAGE: { + sqlite3_result_int(ctx, pCsr->nPage); + break; + } + case BINFO_COLUMN_DEPTH: { + sqlite3_result_int(ctx, pCsr->depth); + break; + } + case BINFO_COLUMN_SCHEMA: { + sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "sqlite_btreeinfo" virtual table module +*/ +int sqlite3BinfoRegister(sqlite3 *db){ + static sqlite3_module binfo_module = { + 0, /* iVersion */ + 0, /* xCreate */ + binfoConnect, /* xConnect */ + binfoBestIndex, /* xBestIndex */ + binfoDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + binfoOpen, /* xOpen - open a cursor */ + binfoClose, /* xClose - close a cursor */ + binfoFilter, /* xFilter - configure scan constraints */ + binfoNext, /* xNext - advance a cursor */ + binfoEof, /* xEof - check for end of scan */ + binfoColumn, /* xColumn - read data */ + binfoRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + }; + return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_btreeinfo_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3BinfoRegister(db); +} diff --git a/manifest b/manifest index d4ab8e50ee..c05d3b0728 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Force\sthe\sautoconf\smakefile\sfor\sMSVC\sto\salways\suse\sthe\s'find.exe'\sincluded\swith\sWindows. -D 2017-10-24T21:17:12.950 +C Add\sthe\ssqlite_btreeinfo\seponymous-only\stable\sfor\sintrospecting\sthe\sschema\nand\sestimating\sthe\ssizes\sof\svarious\sbtrees. +D 2017-10-25T01:34:24.293 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -258,6 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb +F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 w ext/misc/btreelist.c F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -1664,7 +1665,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441 -R 54e49932e96b845a01a6eb1a3cc08724 -U mistachkin -Z 580a702e0e85c423f77f3c922262f566 +P b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887 +R 2cb01c8a7e418d5768c813428fd16a20 +U drh +Z 2d2691251ae11d16de493df9f4384426 diff --git a/manifest.uuid b/manifest.uuid index 5f2c913791..fb6118cf93 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887 \ No newline at end of file +1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d \ No newline at end of file From b5d013edd1f7fe1623b5951bb4664f6b23638f2f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Oct 2017 16:14:12 +0000 Subject: [PATCH 040/156] Do not reference the ioctl() system call in the unix backend unless it is actually needed by the Batch Atomic Write extension. This should allow the build to work on VxWorks. FossilOrigin-Name: adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c05d3b0728..29877dd0e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite_btreeinfo\seponymous-only\stable\sfor\sintrospecting\sthe\sschema\nand\sestimating\sthe\ssizes\sof\svarious\sbtrees. -D 2017-10-25T01:34:24.293 +C Do\snot\sreference\sthe\sioctl()\ssystem\scall\sin\sthe\sunix\sbackend\sunless\sit\nis\sactually\sneeded\sby\sthe\sBatch\sAtomic\sWrite\sextension.\s\sThis\sshould\sallow\nthe\sbuild\sto\swork\son\sVxWorks. +D 2017-10-25T16:14:12.910 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -258,7 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 w ext/misc/btreelist.c +F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -446,7 +446,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b +F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1665,7 +1665,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887 -R 2cb01c8a7e418d5768c813428fd16a20 +P 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d +R 15e40e111eafb5e48910635a49cabea3 U drh -Z 2d2691251ae11d16de493df9f4384426 +Z 02638cae8d6ec29df692272f34f9ba5b diff --git a/manifest.uuid b/manifest.uuid index fb6118cf93..b582a39fb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d \ No newline at end of file +adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 4445104dd6..c33d21fd9b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -513,7 +513,11 @@ static struct unix_syscall { #endif #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#else + { "ioctl", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) }; /* End of the overrideable system calls */ From 1917e92fdb0084faceb44dc06694aa13b051843b Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Oct 2017 16:38:34 +0000 Subject: [PATCH 041/156] Add SQL scalar function rtreecheck() to the rtree module. For running checks to ensure the shadow tables used by an rtree virtual table are internally consistent. FossilOrigin-Name: dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327 --- ext/rtree/rtree.c | 424 +++++++++++++++++++++++++++++++++++++- ext/rtree/rtree1.test | 2 +- ext/rtree/rtree2.test | 8 +- ext/rtree/rtree4.test | 2 + ext/rtree/rtree5.test | 2 + ext/rtree/rtree7.test | 3 + ext/rtree/rtree8.test | 6 +- ext/rtree/rtree9.test | 6 +- ext/rtree/rtreeA.test | 26 ++- ext/rtree/rtreeB.test | 3 + ext/rtree/rtreeC.test | 2 + ext/rtree/rtreeE.test | 5 +- ext/rtree/rtreeF.test | 3 + ext/rtree/rtreeG.test | 2 + ext/rtree/rtree_util.tcl | 5 + ext/rtree/rtreecheck.test | 113 ++++++++++ manifest | 43 ++-- manifest.uuid | 2 +- 18 files changed, 623 insertions(+), 34 deletions(-) create mode 100644 ext/rtree/rtreecheck.test diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 32b9b3027a..500d922102 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -209,7 +209,7 @@ struct RtreeSearchPoint { ** The smallest possible node-size is (512-64)==448 bytes. And the largest ** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). ** Therefore all non-root nodes must contain at least 3 entries. Since -** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 3^40 is greater than 2^64, an r-tree structure always has a depth of ** 40 or less. */ #define RTREE_MAX_DEPTH 40 @@ -3608,6 +3608,425 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ } } +/* +** Context object passed between the various routines that make up the +** implementation of integrity-check function rtreecheck(). +*/ +typedef struct RtreeCheck RtreeCheck; +struct RtreeCheck { + sqlite3 *db; /* Database handle */ + const char *zDb; /* Database containing rtree table */ + const char *zTab; /* Name of rtree table */ + int bInt; /* True for rtree_i32 table */ + int nDim; /* Number of dimensions for this rtree tbl */ + sqlite3_stmt *pGetNode; /* Statement used to retrieve nodes */ + sqlite3_stmt *aCheckMapping[2]; /* Statements to query %_parent/%_rowid */ + int nLeaf; /* Number of leaf cells in table */ + int nNonLeaf; /* Number of non-leaf cells in table */ + int rc; /* Return code */ + char *zReport; /* Message to report */ + int nErr; /* Number of lines in zReport */ +}; + +#define RTREE_CHECK_MAX_ERROR 100 + +/* +** Reset SQL statement pStmt. If the sqlite3_reset() call returns an error, +** and RtreeCheck.rc==SQLITE_OK, set RtreeCheck.rc to the error code. +*/ +static void rtreeCheckReset(RtreeCheck *pCheck, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( pCheck->rc==SQLITE_OK ) pCheck->rc = rc; +} + +/* +** The second and subsequent arguments to this function are a format string +** and printf style arguments. This function formats the string and attempts +** to compile it as an SQL statement. +** +** If successful, a pointer to the new SQL statement is returned. Otherwise, +** NULL is returned and an error code left in RtreeCheck.rc. +*/ +static sqlite3_stmt *rtreeCheckPrepare( + RtreeCheck *pCheck, /* RtreeCheck object */ + const char *zFmt, ... /* Format string and trailing args */ +){ + va_list ap; + va_start(ap, zFmt); + char *z = sqlite3_vmprintf(zFmt, ap); + sqlite3_stmt *pRet = 0; + + if( pCheck->rc==SQLITE_OK ){ + pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + } + + sqlite3_free(z); + va_end(ap); + return pRet; +} + +/* +** The second and subsequent arguments to this function are a printf() +** style format string and arguments. This function formats the string and +** appends it to the report being accumuated in pCheck. +*/ +static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pCheck->rc==SQLITE_OK && pCheck->nErrrc = SQLITE_NOMEM; + }else{ + pCheck->zReport = sqlite3_mprintf("%z%s%z", + pCheck->zReport, (pCheck->zReport ? "\n" : ""), z + ); + if( pCheck->zReport==0 ){ + pCheck->rc = SQLITE_NOMEM; + } + } + pCheck->nErr++; + } + va_end(ap); +} + +/* +** This function is a no-op if there is already an error code stored +** in the RtreeCheck object indicated by the first argument. NULL is +** returned in this case. +** +** Otherwise, the contents of rtree table node iNode are loaded from +** the database and copied into a buffer obtained from sqlite3_malloc(). +** If no error occurs, a pointer to the buffer is returned and (*pnNode) +** is set to the size of the buffer in bytes. +** +** Or, if an error does occur, NULL is returned and an error code left +** in the RtreeCheck object. The final value of *pnNode is undefined in +** this case. +*/ +static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){ + u8 *pRet = 0; /* Return value */ + + assert( pCheck->rc==SQLITE_OK ); + if( pCheck->pGetNode==0 ){ + pCheck->pGetNode = rtreeCheckPrepare(pCheck, + "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", + pCheck->zDb, pCheck->zTab + ); + } + + if( pCheck->rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck->pGetNode, 1, iNode); + if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){ + int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0); + const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0); + pRet = sqlite3_malloc(nNode); + if( pRet==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + memcpy(pRet, pNode, nNode); + *pnNode = nNode; + } + } + rtreeCheckReset(pCheck, pCheck->pGetNode); + if( pCheck->rc==SQLITE_OK && pRet==0 ){ + rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode); + } + } + + return pRet; +} + +/* +** This function is used to check that the %_parent (if bLeaf==0) or %_rowid +** (if bLeaf==1) table contains a specified entry. The schemas of the +** two tables are: +** +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** In both cases, this function checks that there exists an entry with +** IPK value iKey and the second column set to iVal. +** +*/ +static void rtreeCheckMapping( + RtreeCheck *pCheck, /* RtreeCheck object */ + int bLeaf, /* True for a leaf cell, false for interior */ + i64 iKey, /* Key for mapping */ + i64 iVal /* Expected value for mapping */ +){ + int rc; + sqlite3_stmt *pStmt; + const char *azSql[2] = { + "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?", + "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?" + }; + + assert( bLeaf==0 || bLeaf==1 ); + if( pCheck->aCheckMapping[bLeaf]==0 ){ + pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck, + azSql[bLeaf], pCheck->zDb, pCheck->zTab + ); + } + if( pCheck->rc!=SQLITE_OK ) return; + + pStmt = pCheck->aCheckMapping[bLeaf]; + sqlite3_bind_int64(pStmt, 1, iKey); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_DONE ){ + rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table", + iKey, iVal, (bLeaf ? "%_rowid" : "%_parent") + ); + }else if( rc==SQLITE_ROW ){ + i64 ii = sqlite3_column_int64(pStmt, 0); + if( ii!=iVal ){ + rtreeCheckAppendMsg(pCheck, + "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)", + iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal + ); + } + } + rtreeCheckReset(pCheck, pStmt); +} + +static void rtreeCheckCellCoord( + RtreeCheck *pCheck, + i64 iNode, + int iCell, + u8 *pCell, /* Pointer to cell coordinates */ + u8 *pParent /* Pointer to parent coordinates */ +){ + RtreeCoord c1, c2; + RtreeCoord p1, p2; + int i; + + for(i=0; inDim; i++){ + readCoord(&pCell[4*2*i], &c1); + readCoord(&pCell[4*(2*i + 1)], &c2); + + /* printf("%e, %e\n", c1.u.f, c2.u.f); */ + if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode + ); + } + + if( pParent ){ + readCoord(&pParent[4*2*i], &p1); + readCoord(&pParent[4*(2*i + 1)], &p2); + + if( (pCheck->bInt ? c1.ibInt ? c2.i>p2.i : c2.f>p2.f) + ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt relative to parent" + , i, iCell, iNode + ); + } + } + } +} + +static void rtreeCheckNode( + RtreeCheck *pCheck, + int iDepth, /* Depth of iNode (0==leaf) */ + u8 *aParent, /* Buffer containing parent coords */ + i64 iNode /* Node to check */ +){ + u8 *aNode = 0; + int nNode = 0; + + assert( iNode==1 || aParent!=0 ); + assert( pCheck->nDim>0 ); + + aNode = rtreeCheckGetNode(pCheck, iNode, &nNode); + if( aNode ){ + if( nNode<4 ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small (%d bytes)", iNode, nNode + ); + }else{ + int nCell; /* Number of cells on page */ + int i; /* Used to iterate through cells */ + if( aParent==0 ){ + iDepth = readInt16(aNode); + if( iDepth>RTREE_MAX_DEPTH ){ + rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth); + sqlite3_free(aNode); + return; + } + } + nCell = readInt16(&aNode[2]); + if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small for cell count of %d (%d bytes)", + iNode, nCell, nNode + ); + } + for(i=0; inDim*2*4)]; + i64 iVal = readInt64(pCell); + rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); + + if( iDepth>0 ){ + rtreeCheckMapping(pCheck, 0, iVal, iNode); + rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); + pCheck->nNonLeaf++; + }else{ + rtreeCheckMapping(pCheck, 1, iVal, iNode); + pCheck->nLeaf++; + } + } + } + sqlite3_free(aNode); + } +} + +static void rtreeCheckCount( + RtreeCheck *pCheck, const char *zTbl, i64 nExpected +){ + if( pCheck->rc==SQLITE_OK ){ + sqlite3_stmt *pCount; + pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'", + pCheck->zDb, pCheck->zTab, zTbl + ); + if( pCount ){ + if( sqlite3_step(pCount)==SQLITE_ROW ){ + i64 nActual = sqlite3_column_int64(pCount, 0); + if( nActual!=nExpected ){ + rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table" + " - expected %lld, actual %lld" , zTbl, nExpected, nActual + ); + } + } + pCheck->rc = sqlite3_finalize(pCount); + } + } +} + +static int rtreeCheck( + sqlite3 *db, /* Database handle to access db through */ + const char *zDb, /* Name of db ("main", "temp" etc.) */ + const char *zTab, /* Name of rtree table to check */ + char **pzReport /* OUT: sqlite3_malloc'd report text */ +){ + RtreeCheck check; /* Common context for various routines */ + sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ + int bEnd = 0; /* True if transaction should be closed */ + + /* Initialize the context object */ + memset(&check, 0, sizeof(check)); + check.db = db; + check.zDb = zDb; + check.zTab = zTab; + + /* If there is not already an open transaction, open one now. This is + ** to ensure that the queries run as part of this integrity-check operate + ** on a consistent snapshot. */ + if( sqlite3_get_autocommit(db) ){ + check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + bEnd = 1; + } + + /* Find number of dimensions in the rtree table. */ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.%Q", zDb, zTab); + if( pStmt ){ + int rc; + check.nDim = (sqlite3_column_count(pStmt) - 1) / 2; + if( check.nDim<1 ){ + rtreeCheckAppendMsg(&check, "Schema corrupt or not an rtree"); + }else if( SQLITE_ROW==sqlite3_step(pStmt) ){ + check.bInt = (sqlite3_column_type(pStmt, 1)==SQLITE_INTEGER); + } + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_CORRUPT ) check.rc = rc; + } + + /* Do the actual integrity-check */ + if( check.rc==SQLITE_OK ){ + rtreeCheckNode(&check, 0, 0, 1); + } + rtreeCheckCount(&check, "_rowid", check.nLeaf); + rtreeCheckCount(&check, "_parent", check.nNonLeaf); + + /* Finalize SQL statements used by the integrity-check */ + sqlite3_finalize(check.pGetNode); + sqlite3_finalize(check.aCheckMapping[0]); + sqlite3_finalize(check.aCheckMapping[1]); + + /* If one was opened, close the transaction */ + if( bEnd ){ + int rc = sqlite3_exec(db, "END", 0, 0, 0); + if( check.rc==SQLITE_OK ) check.rc = rc; + } + *pzReport = check.zReport; + return check.rc; +} + +/* +** Usage: +** +** rtreecheck(); +** rtreecheck(, ); +** +** Invoking this SQL function runs an integrity-check on the named rtree +** table. The integrity-check verifies the following: +** +** 1. For each cell in the r-tree structure (%_node table), that: +** +** a) for each dimension, (coord1 <= coord2). +** +** b) unless the cell is on the root node, that the cell is bounded +** by the parent cell on the parent node. +** +** c) for leaf nodes, that there is an entry in the %_rowid +** table corresponding to the cell's rowid value that +** points to the correct node. +** +** d) for cells on non-leaf nodes, that there is an entry in the +** %_parent table mapping from the cell's child node to the +** node that it resides on. +** +** 2. That there are the same number of entries in the %_rowid table +** as there are leaf cells in the r-tree structure, and that there +** is a leaf cell that corresponds to each entry in the %_rowid table. +** +** 3. That there are the same number of entries in the %_parent table +** as there are non-leaf cells in the r-tree structure, and that +** there is a non-leaf cell that corresponds to each entry in the +** %_parent table. +*/ +static void rtreecheck( + sqlite3_context *ctx, + int nArg, + sqlite3_value **apArg +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(ctx, + "wrong number of arguments to function rtreecheck()", -1 + ); + }else{ + int rc; + char *zReport = 0; + const char *zDb = (const char*)sqlite3_value_text(apArg[0]); + const char *zTab; + if( nArg==1 ){ + zTab = zDb; + zDb = "main"; + }else{ + zTab = (const char*)sqlite3_value_text(apArg[1]); + } + rc = rtreeCheck(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); + if( rc==SQLITE_OK ){ + sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(ctx, rc); + } + sqlite3_free(zReport); + } +} + + /* ** Register the r-tree module with database handle db. This creates the ** virtual table module "rtree" and the debugging/analysis scalar @@ -3621,6 +4040,9 @@ int sqlite3RtreeInit(sqlite3 *db){ if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtreecheck", -1, utf8, 0,rtreecheck, 0,0); + } if( rc==SQLITE_OK ){ #ifdef SQLITE_RTREE_INT_ONLY void *c = (void *)RTREE_COORD_INT32; diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index becc8e1a97..0deee6635b 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -519,7 +519,7 @@ foreach {tn sql_template testdata} { do_test $testname.2 [list sql_uses_stmt db $sql] $uses do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data - do_test $testname.4 { rtree_check db t1 } 0 + do_rtree_integrity_test $testname.4 t1 db close } } diff --git a/ext/rtree/rtree2.test b/ext/rtree/rtree2.test index f5d15cc6b9..853737b9c7 100644 --- a/ext/rtree/rtree2.test +++ b/ext/rtree/rtree2.test @@ -81,9 +81,7 @@ foreach module {rtree_i32 rtree} { set rc } {1} - do_test rtree2-$module.$nDim.3 { - rtree_check db t1 - } 0 + do_rtree_integrity_test rtree2-$module.$nDim.3 t1 set OPS [list < > <= >= =] for {set ii 0} {$ii < $::NSELECT} {incr ii} { @@ -133,9 +131,7 @@ foreach module {rtree_i32 rtree} { } set rc } {1} - do_test rtree2-$module.$nDim.5.$ii.2 { - rtree_check db t1 - } {0} + do_rtree_integrity_test rtree2-$module.$nDim.5.$ii.2 t1 } do_test rtree2-$module.$nDim.6 { diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test index a3872b0735..af3f8d3995 100644 --- a/ext/rtree/rtree4.test +++ b/ext/rtree/rtree4.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { @@ -246,6 +247,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} { } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]] } + do_rtree_integrity_test rtree4-$nDim.3 rx } finish_test diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 8ff90b0cb7..749385e882 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -16,6 +16,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { @@ -76,5 +77,6 @@ do_test rtree5-1.13 { y1=-2147483648 AND y2=-2147483643 } } {2 2147483643 2147483647 -2147483648 -2147483643} +do_rtree_integrity_test rtree5-1.14 t1 finish_test diff --git a/ext/rtree/rtree7.test b/ext/rtree/rtree7.test index 4eee4c219a..1556179c4f 100644 --- a/ext/rtree/rtree7.test +++ b/ext/rtree/rtree7.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree||!vacuum { @@ -67,4 +68,6 @@ do_test rtree7-1.5 { } } {51 102 153 204} +do_rtree_integrity_test rtree7-1.6 rt + finish_test diff --git a/ext/rtree/rtree8.test b/ext/rtree/rtree8.test index 024d098e07..ac18fb477d 100644 --- a/ext/rtree/rtree8.test +++ b/ext/rtree/rtree8.test @@ -14,6 +14,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } @@ -64,6 +65,7 @@ do_test rtree8-1.2.2 { nested_select 1 } {51} # nodes internally. # populate_t1 1500 +do_rtree_integrity_test rtree8-1.3.0 t1 do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {164} do_test rtree8-1.3.2 { set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}] @@ -158,13 +160,15 @@ do_test rtree8-5.2 { } execsql COMMIT } {} -do_test rtree8-5.3 { +do_rtree_integrity_test rtree8-5.3 t2 +do_test rtree8-5.4 { execsql BEGIN for {set i 0} {$i < 200} {incr i} { execsql { DELETE FROM t2 WHERE id = $i } } execsql COMMIT } {} +do_rtree_integrity_test rtree8-5.5 t2 finish_test diff --git a/ext/rtree/rtree9.test b/ext/rtree/rtree9.test index b2361b2bd0..a7cd344d76 100644 --- a/ext/rtree/rtree9.test +++ b/ext/rtree/rtree9.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } ifcapable rtree_int_only { finish_test; return } @@ -42,6 +43,7 @@ for {set i 0} {$i < 1000} {incr i} { set z [expr ($i/100)%10] execsql { INSERT INTO rt VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) } } +do_rtree_integrity_test rtree9-2.0 rt do_execsql_test rtree9-2.1 { SELECT id FROM rt WHERE id MATCH cube(2.5, 2.5, 2.5, 1, 1, 1) ORDER BY id; } {222 223 232 233 322 323 332 333} @@ -50,7 +52,7 @@ do_execsql_test rtree9-2.2 { } {555 556 565 566 655 656 665 666} -do_execsql_test rtree9-3.1 { +do_execsql_test rtree9-3.0 { CREATE VIRTUAL TABLE rt32 USING rtree_i32(id, x1, x2, y1, y2, z1, z2); } {} for {set i 0} {$i < 1000} {incr i} { @@ -59,6 +61,7 @@ for {set i 0} {$i < 1000} {incr i} { set z [expr ($i/100)%10] execsql { INSERT INTO rt32 VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) } } +do_rtree_integrity_test rtree9-3.1 rt32 do_execsql_test rtree9-3.2 { SELECT id FROM rt32 WHERE id MATCH cube(3, 3, 3, 1, 1, 1) ORDER BY id; } {222 223 224 232 233 234 242 243 244 322 323 324 332 333 334 342 343 344 422 423 424 432 433 434 442 443 444} @@ -121,5 +124,6 @@ do_execsql_test rtree9-5.3 { UPDATE rt2 SET xmin=xmin+5, ymin=ymin+5, xmax=xmax+5, ymax=ymax+5; SELECT id FROM rt2 WHERE id MATCH circle(5.0, 5.0, 2.0); } {1 2 3 4 13 14 15 16 17} +do_rtree_integrity_test rtree9-5.4 rt2 finish_test diff --git a/ext/rtree/rtreeA.test b/ext/rtree/rtreeA.test index d583a15264..190d7d7532 100644 --- a/ext/rtree/rtreeA.test +++ b/ext/rtree/rtreeA.test @@ -108,6 +108,12 @@ do_corruption_tests rtreeA-1.1 { 4 "SELECT * FROM t1 WHERE x1<10 AND x2>12" } +do_execsql_test rtreeA-1.1.1 { + SELECT rtreecheck('main', 't1') +} {{Node 1 missing from database +Wrong number of entries in %_rowid table - expected 0, actual 500 +Wrong number of entries in %_parent table - expected 0, actual 23}} + do_execsql_test rtreeA-1.2.0 { DROP TABLE t1_node } {} do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" { 1 "SELECT * FROM t1" @@ -157,6 +163,10 @@ do_corruption_tests rtreeA-3.1 { 3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)" } +do_execsql_test rtreeA-3.1.0.3 { + SELECT rtreecheck('main', 't1')!="ok" +} {1} + do_test rtreeA-3.2.0 { set_tree_depth t1 1000 } {1000} do_corruption_tests rtreeA-3.2 { 1 "SELECT * FROM t1" @@ -176,6 +186,12 @@ do_corruption_tests rtreeA-3.3 { 3 "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)" } +do_execsql_test rtreeA-3.3.3.4 { + SELECT rtreecheck('main', 't1') +} {{Rtree depth out of range (65535) +Wrong number of entries in %_rowid table - expected 0, actual 499 +Wrong number of entries in %_parent table - expected 0, actual 23}} + #------------------------------------------------------------------------- # Set the "number of entries" field on some nodes incorrectly. # @@ -203,6 +219,10 @@ do_corruption_tests rtreeA-5.1 { 2 "DELETE FROM t1" } +do_execsql_test rtreeA-5.2 { + SELECT rtreecheck('main', 't1')!="ok" +} {1} + #------------------------------------------------------------------------- # Add some bad entries to the %_parent table. # @@ -216,6 +236,10 @@ do_corruption_tests rtreeA-6.1 { 2 "UPDATE t1 SET x1=x1+1, x2=x2+1" } +do_execsql_test rtreeA-6.2 { + SELECT rtreecheck('main', 't1')!="ok" +} {1} + #------------------------------------------------------------------------- # Truncated blobs in the _node table. # @@ -233,5 +257,5 @@ do_test rtreeA-7.120 { } {SQLITE_CORRUPT_VTAB} - finish_test + diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test index aeb308eca7..6fc31042ca 100644 --- a/ext/rtree/rtreeB.test +++ b/ext/rtree/rtreeB.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } @@ -44,4 +45,6 @@ ifcapable rtree_int_only { } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}} } +do_rtree_integrity_test rtreeB-1.2 t1 + finish_test diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index a26c401e0d..a5e5b97f96 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } set testprefix rtreeC @@ -180,6 +181,7 @@ do_execsql_test 5.1 { INSERT INTO rt SELECT x, x, x+1 FROM t1 WHERE x<=5; } +do_rtree_integrity_test 5.1.1 rt # First test a query with no ANALYZE data at all. The outer loop is # real table "t1". diff --git a/ext/rtree/rtreeE.test b/ext/rtree/rtreeE.test index 3e5ba3a67f..72dcc94c9f 100644 --- a/ext/rtree/rtreeE.test +++ b/ext/rtree/rtreeE.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } ifcapable rtree_int_only { finish_test; return } @@ -25,7 +26,7 @@ ifcapable rtree_int_only { finish_test; return } # register_circle_geom db -do_execsql_test rtreeE-1.1 { +do_execsql_test rtreeE-1.0.0 { PRAGMA page_size=512; CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1); @@ -47,6 +48,7 @@ do_execsql_test rtreeE-1.1 { y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) INSERT INTO rt1 SELECT 200+x+5*y, x*7, x*7+15, y*7+200, y*7+215 FROM x, y; } {} +do_rtree_integrity_test rtreeE-1.0.1 rt1 # Queries against each of the three clusters */ do_execsql_test rtreeE-1.1 { @@ -111,6 +113,7 @@ do_test rtreeE-2.1 { COMMIT; } } {} +do_rtree_integrity_test rtreeE-2.1.1 rt2 for {set i 1} {$i<=200} {incr i} { set dx [expr {int(rand()*100)}] diff --git a/ext/rtree/rtreeF.test b/ext/rtree/rtreeF.test index c9620d34f7..561770d085 100644 --- a/ext/rtree/rtreeF.test +++ b/ext/rtree/rtreeF.test @@ -28,6 +28,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } @@ -78,4 +79,6 @@ do_execsql_test rtreeF-1.5 { SELECT y FROM t2 ORDER BY y; } {1 4 5 | 1 4} +do_rtree_integrity_test rtreeF-1.6 t3 + finish_test diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test index bffd17ecaa..3bef89c8e7 100644 --- a/ext/rtree/rtreeG.test +++ b/ext/rtree/rtreeG.test @@ -15,6 +15,7 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } +source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl ifcapable !rtree { finish_test ; return } @@ -37,6 +38,7 @@ do_execsql_test rtreeG-1.2 { INSERT INTO t1 VALUES(1,10,15,5,23),(2,20,21,5,23),(3,10,15,20,30); SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25; } {1} +do_rtree_integrity_test rtreeG-1.2.integrity t1 do_test rtreeG-1.2log { set ::log } {} diff --git a/ext/rtree/rtree_util.tcl b/ext/rtree/rtree_util.tcl index 50a1b58065..afa588e4e9 100644 --- a/ext/rtree/rtree_util.tcl +++ b/ext/rtree/rtree_util.tcl @@ -190,3 +190,8 @@ proc rtree_treedump {db zTab} { set d [rtree_depth $db $zTab] rtree_nodetreedump $db $zTab "" $d 1 } + +proc do_rtree_integrity_test {tn tbl} { + uplevel [list do_execsql_test $tn "SELECT rtreecheck('$tbl')" ok] +} + diff --git a/ext/rtree/rtreecheck.test b/ext/rtree/rtreecheck.test new file mode 100644 index 0000000000..eb0e8f0acb --- /dev/null +++ b/ext/rtree/rtreecheck.test @@ -0,0 +1,113 @@ +# 2017 August 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix rtreecheck + +ifcapable !rtree { + finish_test + return +} + +proc swap_int32 {blob i0 i1} { + binary scan $blob I* L + + set a [lindex $L $i0] + set b [lindex $L $i1] + + lset L $i0 $b + lset L $i1 $a + + binary format I* $L +} + +do_catchsql_test 1.0 { + SELECT rtreecheck(); +} {1 {wrong number of arguments to function rtreecheck()}} + +do_catchsql_test 1.1 { + SELECT rtreecheck(0,0,0); +} {1 {wrong number of arguments to function rtreecheck()}} + + +proc setup_simple_db {{module rtree}} { + reset_db + db func swap_int32 swap_int32 + execsql " + CREATE VIRTUAL TABLE r1 USING $module (id, x1, x2, y1, y2); + INSERT INTO r1 VALUES(1, 5, 5, 5, 5); -- 3 + INSERT INTO r1 VALUES(2, 6, 6, 6, 6); -- 9 + INSERT INTO r1 VALUES(3, 7, 7, 7, 7); -- 15 + INSERT INTO r1 VALUES(4, 8, 8, 8, 8); -- 21 + INSERT INTO r1 VALUES(5, 9, 9, 9, 9); -- 27 + " +} + +setup_simple_db +do_execsql_test 2.1 { + SELECT rtreecheck('r1') +} {ok} + +do_execsql_test 2.2 { + UPDATE r1_node SET data = swap_int32(data, 3, 9); + UPDATE r1_node SET data = swap_int32(data, 23, 29); +} + +do_execsql_test 2.3 { + SELECT rtreecheck('r1') +} {{Dimension 0 of cell 0 on node 1 is corrupt +Dimension 1 of cell 3 on node 1 is corrupt}} + +setup_simple_db +do_execsql_test 2.4 { + DELETE FROM r1_rowid WHERE rowid = 3; + SELECT rtreecheck('r1') +} {{Mapping (3 -> 1) missing from %_rowid table +Wrong number of entries in %_rowid table - expected 5, actual 4}} + +setup_simple_db +do_execsql_test 2.5 { + UPDATE r1_rowid SET nodeno=2 WHERE rowid=3; + SELECT rtreecheck('r1') +} {{Found (3 -> 2) in %_rowid table, expected (3 -> 1)}} + +################ +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE r1 USING rtree_i32(id, x1, x2); + INSERT INTO r1 VALUES(1, 0x7FFFFFFF*-1, 0x7FFFFFFF); + INSERT INTO r1 VALUES(2, 0x7FFFFFFF*-1, 5); + INSERT INTO r1 VALUES(3, -5, 5); + INSERT INTO r1 VALUES(4, 5, 0x11111111); + INSERT INTO r1 VALUES(5, 5, 0x00800000); + INSERT INTO r1 VALUES(6, 5, 0x00008000); + INSERT INTO r1 VALUES(7, 5, 0x00000080); + INSERT INTO r1 VALUES(8, 5, 0x40490fdb); + INSERT INTO r1 VALUES(9, 0x7f800000, 0x7f900000); + SELECT rtreecheck('r1') +} {ok} + +breakpoint +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE r2 USING rtree_i32(id, x1, x2); + INSERT INTO r2 VALUES(2, -1*(1<<31), -1*(1<<31)+5); + SELECT rtreecheck('r2') +} {ok} + + +finish_test + diff --git a/manifest b/manifest index 29877dd0e2..57d11641e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreference\sthe\sioctl()\ssystem\scall\sin\sthe\sunix\sbackend\sunless\sit\nis\sactually\sneeded\sby\sthe\sBatch\sAtomic\sWrite\sextension.\s\sThis\sshould\sallow\nthe\sbuild\sto\swork\son\sVxWorks. -D 2017-10-25T16:14:12.910 +C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\nto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\nconsistent. +D 2017-10-25T16:38:34.144 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -329,26 +329,27 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb +F ext/rtree/rtree.c 7941c4283bef9ecdd1b7a5c20122006b7d363836468fe7f4bef46698c55d1d93 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test 4fdd60ae034e43f2fefc26492032d02e742e8b14d468b7c51d95a1e2fa47cf00 -F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba +F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436 +F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482 -F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 -F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e +F ext/rtree/rtree4.test 67b021858ba4334c8d49b3449476942c2ce0e5ef7123538f2e9dd508ed03a12d +F ext/rtree/rtree5.test 8aaa4bcdc42f718fe165572f5623e4732831aca95a2bc32482d33d4d2cf1325d F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 -F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 -F ext/rtree/rtree8.test 076d9d5b783b61b7a23a5ab45fc899551dfffd821974f36ee599ff29f4de7a61 -F ext/rtree/rtree9.test 8bfa84dfaba1c897468a2448c28db0a00ad12d464225b5993c7814e907f3776f -F ext/rtree/rtreeA.test c09ad3f76c08feac00770685ff50ca12966dc0c641bf19a982b26a80643b46d1 -F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e -F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d +F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5 +F ext/rtree/rtree8.test 649f5a37ec656028a4a32674b9b1183104285a7625a09d2a8f52a1cef72c93f2 +F ext/rtree/rtree9.test c646f12c8c1c68ef015c6c043d86a0c42488e2e68ed1bb1b0771a7ca246cbabf +F ext/rtree/rtreeA.test 20623ca337ca3bd7e008cc9fb49e44dbe97f1a80b238e10a12bb4afcd0da3776 +F ext/rtree/rtreeB.test 4cec297f8e5c588654bbf3c6ed0903f10612be8a2878055dd25faf8c71758bc9 +F ext/rtree/rtreeC.test d9d06dda1aee68b4dc227dfcc899f335f8b621e9d1920ee3d4e5dab8ccd71db7 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc -F ext/rtree/rtreeE.test 45a147a64a76306172819562309681d8e90f94bb -F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4 -F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1 +F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9 +F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331 +F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 -F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea +F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed +F ext/rtree/rtreecheck.test f610cb77ca1ba611e656018a7d960cd46054baecd2f12d1149bf1fec121aa230 F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de @@ -1665,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d -R 15e40e111eafb5e48910635a49cabea3 -U drh -Z 02638cae8d6ec29df692272f34f9ba5b +P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 +R b2ab5ba3cbab5da53d273c6ee124bea0 +U dan +Z 3064e6e3a20de6115706024077565624 diff --git a/manifest.uuid b/manifest.uuid index b582a39fb7..d2644e0a82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 \ No newline at end of file +dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327 \ No newline at end of file From 8c66e5b73e0424ada2b96e340058be96b80c2860 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 25 Oct 2017 18:00:58 +0000 Subject: [PATCH 042/156] Fix compilation issue (C99-ism) seen with MSVC. FossilOrigin-Name: 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5 --- ext/rtree/rtree.c | 6 ++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 500d922102..c7bf671c96 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3652,10 +3652,12 @@ static sqlite3_stmt *rtreeCheckPrepare( const char *zFmt, ... /* Format string and trailing args */ ){ va_list ap; - va_start(ap, zFmt); - char *z = sqlite3_vmprintf(zFmt, ap); + char *z; sqlite3_stmt *pRet = 0; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + if( pCheck->rc==SQLITE_OK ){ pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); } diff --git a/manifest b/manifest index 57d11641e5..9fac80b2d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\nto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\nconsistent. -D 2017-10-25T16:38:34.144 +C Fix\scompilation\sissue\s(C99-ism)\sseen\swith\sMSVC. +D 2017-10-25T18:00:58.356 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 7941c4283bef9ecdd1b7a5c20122006b7d363836468fe7f4bef46698c55d1d93 +F ext/rtree/rtree.c 49ced71763c4bd170ceb0b46ab34c749f3356a8c0e36ce6b9e2f36dc63716609 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 -R b2ab5ba3cbab5da53d273c6ee124bea0 -U dan -Z 3064e6e3a20de6115706024077565624 +P dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327 +R bbf1f0574f080f1a74b750b54fc22d8d +U mistachkin +Z e8d8b6c6557c63f0494992168a2a41d0 diff --git a/manifest.uuid b/manifest.uuid index d2644e0a82..b8040f3c9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327 \ No newline at end of file +61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5 \ No newline at end of file From 7e2b38c53b56b2ee025e7e0616bf9c2da3665bea Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Oct 2017 18:17:24 +0000 Subject: [PATCH 043/156] Add tests cases and fix minor issues in the rtreecheck() function. FossilOrigin-Name: d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 --- ext/rtree/rtree.c | 90 +++++++++++++++++++++++++++------------ ext/rtree/rtreecheck.test | 49 ++++++++++++++++++++- manifest | 17 ++++---- manifest.uuid | 2 +- 4 files changed, 119 insertions(+), 39 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index c7bf671c96..ff15a192a1 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3659,7 +3659,11 @@ static sqlite3_stmt *rtreeCheckPrepare( z = sqlite3_vmprintf(zFmt, ap); if( pCheck->rc==SQLITE_OK ){ - pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + if( z==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + } } sqlite3_free(z); @@ -3781,7 +3785,7 @@ static void rtreeCheckMapping( ); }else if( rc==SQLITE_ROW ){ i64 ii = sqlite3_column_int64(pStmt, 0); - if( ii!=iVal ){ + if( ii!=iVal ){ rtreeCheckAppendMsg(pCheck, "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)", iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal @@ -3791,10 +3795,22 @@ static void rtreeCheckMapping( rtreeCheckReset(pCheck, pStmt); } +/* +** Argument pCell points to an array of coordinates stored on an rtree page. +** This function checks that the coordinates are internally consistent (no +** x1>x2 conditions) and adds an error message to the RtreeCheck object +** if they are not. +** +** Additionally, if pParent is not NULL, then it is assumed to point to +** the array of coordinates on the parent page that bound the page +** containing pCell. In this case it is also verified that the two +** sets of coordinates are mutually consistent and an error message added +** to the RtreeCheck object if they are not. +*/ static void rtreeCheckCellCoord( RtreeCheck *pCheck, - i64 iNode, - int iCell, + i64 iNode, /* Node id to use in error messages */ + int iCell, /* Cell number to use in error messages */ u8 *pCell, /* Pointer to cell coordinates */ u8 *pParent /* Pointer to parent coordinates */ ){ @@ -3829,6 +3845,14 @@ static void rtreeCheckCellCoord( } } +/* +** Run rtreecheck() checks on node iNode, which is at depth iDepth within +** the r-tree structure. Argument aParent points to the array of coordinates +** that bound node iNode on the parent node. +** +** If any problems are discovered, an error message is appended to the +** report accumulated in the RtreeCheck object. +*/ static void rtreeCheckNode( RtreeCheck *pCheck, int iDepth, /* Depth of iNode (0==leaf) */ @@ -3864,19 +3888,20 @@ static void rtreeCheckNode( "Node %lld is too small for cell count of %d (%d bytes)", iNode, nCell, nNode ); - } - for(i=0; inDim*2*4)]; - i64 iVal = readInt64(pCell); - rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); + }else{ + for(i=0; inDim*2*4)]; + i64 iVal = readInt64(pCell); + rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); - if( iDepth>0 ){ - rtreeCheckMapping(pCheck, 0, iVal, iNode); - rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); - pCheck->nNonLeaf++; - }else{ - rtreeCheckMapping(pCheck, 1, iVal, iNode); - pCheck->nLeaf++; + if( iDepth>0 ){ + rtreeCheckMapping(pCheck, 0, iVal, iNode); + rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); + pCheck->nNonLeaf++; + }else{ + rtreeCheckMapping(pCheck, 1, iVal, iNode); + pCheck->nLeaf++; + } } } } @@ -3884,9 +3909,14 @@ static void rtreeCheckNode( } } -static void rtreeCheckCount( - RtreeCheck *pCheck, const char *zTbl, i64 nExpected -){ +/* +** The second argument to this function must be either "_rowid" or +** "_parent". This function checks that the number of entries in the +** %_rowid or %_parent table is exactly nExpect. If not, it adds +** an error message to the report in the RtreeCheck object indicated +** by the first argument. +*/ +static void rtreeCheckCount(RtreeCheck *pCheck, const char *zTbl, i64 nExpect){ if( pCheck->rc==SQLITE_OK ){ sqlite3_stmt *pCount; pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'", @@ -3895,9 +3925,9 @@ static void rtreeCheckCount( if( pCount ){ if( sqlite3_step(pCount)==SQLITE_ROW ){ i64 nActual = sqlite3_column_int64(pCount, 0); - if( nActual!=nExpected ){ + if( nActual!=nExpect ){ rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table" - " - expected %lld, actual %lld" , zTbl, nExpected, nActual + " - expected %lld, actual %lld" , zTbl, nExpect, nActual ); } } @@ -3906,7 +3936,11 @@ static void rtreeCheckCount( } } -static int rtreeCheck( +/* +** This function does the bulk of the work for the rtree integrity-check. +** It is called by rtreecheck(), which is the SQL function implementation. +*/ +static int rtreeCheckTable( sqlite3 *db, /* Database handle to access db through */ const char *zDb, /* Name of db ("main", "temp" etc.) */ const char *zTab, /* Name of rtree table to check */ @@ -3945,11 +3979,13 @@ static int rtreeCheck( } /* Do the actual integrity-check */ - if( check.rc==SQLITE_OK ){ - rtreeCheckNode(&check, 0, 0, 1); + if( check.nDim>=1 ){ + if( check.rc==SQLITE_OK ){ + rtreeCheckNode(&check, 0, 0, 1); + } + rtreeCheckCount(&check, "_rowid", check.nLeaf); + rtreeCheckCount(&check, "_parent", check.nNonLeaf); } - rtreeCheckCount(&check, "_rowid", check.nLeaf); - rtreeCheckCount(&check, "_parent", check.nNonLeaf); /* Finalize SQL statements used by the integrity-check */ sqlite3_finalize(check.pGetNode); @@ -4018,7 +4054,7 @@ static void rtreecheck( }else{ zTab = (const char*)sqlite3_value_text(apArg[1]); } - rc = rtreeCheck(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); + rc = rtreeCheckTable(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); if( rc==SQLITE_OK ){ sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); }else{ diff --git a/ext/rtree/rtreecheck.test b/ext/rtree/rtreecheck.test index eb0e8f0acb..cbe8b513c9 100644 --- a/ext/rtree/rtreecheck.test +++ b/ext/rtree/rtreecheck.test @@ -35,6 +35,12 @@ proc swap_int32 {blob i0 i1} { binary format I* $L } +proc set_int32 {blob idx val} { + binary scan $blob I* L + lset L $idx $val + binary format I* $L +} + do_catchsql_test 1.0 { SELECT rtreecheck(); } {1 {wrong number of arguments to function rtreecheck()}} @@ -85,7 +91,6 @@ do_execsql_test 2.5 { SELECT rtreecheck('r1') } {{Found (3 -> 2) in %_rowid table, expected (3 -> 1)}} -################ reset_db do_execsql_test 3.0 { CREATE VIRTUAL TABLE r1 USING rtree_i32(id, x1, x2); @@ -101,13 +106,53 @@ do_execsql_test 3.0 { SELECT rtreecheck('r1') } {ok} -breakpoint do_execsql_test 3.1 { CREATE VIRTUAL TABLE r2 USING rtree_i32(id, x1, x2); INSERT INTO r2 VALUES(2, -1*(1<<31), -1*(1<<31)+5); SELECT rtreecheck('r2') } {ok} +do_execsql_test 3.2 { + BEGIN; + UPDATE r2_node SET data = X'123456'; + SELECT rtreecheck('r2')!="ok"; +} {1} + +do_execsql_test 3.3 { + ROLLBACK; + UPDATE r2_node SET data = X'00001234'; + SELECT rtreecheck('r2')!="ok"; +} {1} + +do_execsql_test 4.0 { + CREATE TABLE notanrtree(i); + SELECT rtreecheck('notanrtree'); +} {{Schema corrupt or not an rtree}} + +#------------------------------------------------------------------------- +# +reset_db +db func set_int32 set_int32 +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE r3 USING rtree_i32(id, x1, x2, y1, y2); + WITH x(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 + ) + INSERT INTO r3 SELECT i, i, i, i, i FROM x; +} +do_execsql_test 5.1 { + BEGIN; + UPDATE r3_node SET data = set_int32(data, 3, 5000); + UPDATE r3_node SET data = set_int32(data, 4, 5000); + SELECT rtreecheck('r3')=='ok' +} 0 +do_execsql_test 5.2 { + ROLLBACK; + BEGIN; + UPDATE r3_node SET data = set_int32(data, 3, 0); + UPDATE r3_node SET data = set_int32(data, 4, 0); + SELECT rtreecheck('r3')=='ok' +} 0 finish_test diff --git a/manifest b/manifest index 1b6fc596a8..367a1a1375 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\sto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\sconsistent. -D 2017-10-25T18:01:58.383 +C Add\stests\scases\sand\sfix\sminor\sissues\sin\sthe\srtreecheck()\sfunction. +D 2017-10-25T18:17:24.834 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 49ced71763c4bd170ceb0b46ab34c749f3356a8c0e36ce6b9e2f36dc63716609 +F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2 @@ -349,7 +349,7 @@ F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed -F ext/rtree/rtreecheck.test f610cb77ca1ba611e656018a7d960cd46054baecd2f12d1149bf1fec121aa230 +F ext/rtree/rtreecheck.test 4d29103d1e16fcbf90135d1c637b833688492b063b2971dfb5dc6ba76555cfee F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de @@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5 -R bbf1f0574f080f1a74b750b54fc22d8d -T +closed 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5 -U mistachkin -Z cbebd749084c1ed38ad0d901e97967c8 +P 7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb +R b18b38b834db7af5e4754d71c435762f +U dan +Z f91c79bfd0a67ab59612bc897b82c53b diff --git a/manifest.uuid b/manifest.uuid index 74306142d2..cb99ca30c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb \ No newline at end of file +d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 \ No newline at end of file From 3cd8aaa74884f2e0db7b8af83870f90e4d460837 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Oct 2017 19:18:33 +0000 Subject: [PATCH 044/156] Fix the sqlite3_dbpage virtual table so that it can read and write from any attached database. FossilOrigin-Name: d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 --- manifest | 16 ++--- manifest.uuid | 2 +- src/dbpage.c | 169 +++++++++++++++++++++++++++++++++++------------ test/dbpage.test | 28 +++++++- 4 files changed, 159 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index 367a1a1375..a593818258 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\scases\sand\sfix\sminor\sissues\sin\sthe\srtreecheck()\sfunction. -D 2017-10-25T18:17:24.834 +C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase. +D 2017-10-25T19:18:33.009 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -413,7 +413,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d +F src/dbpage.c 196a072dffda3ad50bb078da86bb39f1d07cabe4f88e2b12482f4f5ffce73352 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304 @@ -715,7 +715,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbpage.test 10b9e91d07b0892444fff4578706648e955b5fb260218298f838da74f0d9d211 +F test/dbpage.test ace7a8d2d66b0ad043dacc61d206991184c9916de6862bf4ff165a7906a624af F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb -R b18b38b834db7af5e4754d71c435762f -U dan -Z f91c79bfd0a67ab59612bc897b82c53b +P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 +R de06ad2d264283eba96aba95f64ee6f5 +U drh +Z 3b4d77a61659a96790923a30bda00b59 diff --git a/manifest.uuid b/manifest.uuid index cb99ca30c6..ce16a7caf0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 \ No newline at end of file +d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index d21c5b6df1..54fb5beaa7 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -42,17 +42,24 @@ struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ int pgno; /* Current page number */ int mxPgno; /* Last page to visit on this scan */ + Pager *pPager; /* Pager being read/written */ + DbPage *pPage1; /* Page 1 of the database */ + int iDb; /* Index of database to analyze */ + int szPage; /* Size of each page in bytes */ }; struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ - Pager *pPager; /* Pager being read/written */ - int iDb; /* Index of database to analyze */ - int szPage; /* Size of each page in bytes */ - int nPage; /* Number of pages in the file */ }; +/* Columns */ +#define DBPAGE_COLUMN_PGNO 0 +#define DBPAGE_COLUMN_DATA 1 +#define DBPAGE_COLUMN_SCHEMA 2 + + + /* ** Connect to or create a dbpagevfs virtual table. */ @@ -65,19 +72,7 @@ static int dbpageConnect( ){ DbpageTable *pTab = 0; int rc = SQLITE_OK; - int iDb; - if( argc>=4 ){ - Token nm; - sqlite3TokenInit(&nm, (char*)argv[3]); - iDb = sqlite3FindDb(db, &nm); - if( iDb<0 ){ - *pzErr = sqlite3_mprintf("no such schema: %s", argv[3]); - return SQLITE_ERROR; - } - }else{ - iDb = 0; - } rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -87,11 +82,8 @@ static int dbpageConnect( assert( rc==SQLITE_OK || pTab==0 ); if( rc==SQLITE_OK ){ - Btree *pBt = db->aDb[iDb].pBt; memset(pTab, 0, sizeof(DbpageTable)); pTab->db = db; - pTab->iDb = iDb; - pTab->pPager = pBt ? sqlite3BtreePager(pBt) : 0; } *ppVtab = (sqlite3_vtab*)pTab; @@ -109,24 +101,55 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){ /* ** idxNum: ** -** 0 full table scan -** 1 pgno=?1 +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 */ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; - pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + int iPlan = 0; + + /* If there is a schema= constraint, it must be honored. Report a + ** ridiculously large estimated cost if the schema= constraint is + ** unavailable + */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->iColumn!=DBPAGE_COLUMN_SCHEMA ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( !p->usable ){ + /* No solution. Use the default SQLITE_BIG_DBL cost */ + pIdxInfo->estimatedRows = 0x7fffffff; + return SQLITE_OK; + } + iPlan = 2; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + + /* If we reach this point, it means that either there is no schema= + ** constraint (in which case we use the "main" schema) or else the + ** schema constraint was accepted. Lower the estimated cost accordingly + */ + pIdxInfo->estimatedCost = 1.0e6; + + /* Check for constraints against pgno */ for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ pIdxInfo->estimatedRows = 1; pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; pIdxInfo->estimatedCost = 1.0; - pIdxInfo->idxNum = 1; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = iPlan ? 2 : 1; pIdxInfo->aConstraintUsage[i].omit = 1; + iPlan |= 1; break; } } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn<=0 && pIdxInfo->aOrderBy[0].desc==0 @@ -160,6 +183,7 @@ static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ */ static int dbpageClose(sqlite3_vtab_cursor *pCursor){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); sqlite3_free(pCsr); return SQLITE_OK; } @@ -179,6 +203,16 @@ static int dbpageEof(sqlite3_vtab_cursor *pCursor){ return pCsr->pgno > pCsr->mxPgno; } +/* +** idxNum: +** +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 +** +** idxStr is not used +*/ static int dbpageFilter( sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, @@ -187,22 +221,40 @@ static int dbpageFilter( DbpageCursor *pCsr = (DbpageCursor *)pCursor; DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; int rc = SQLITE_OK; - Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + sqlite3 *db = pTab->db; + Btree *pBt; - pTab->szPage = sqlite3BtreeGetPageSize(pBt); - pTab->nPage = sqlite3BtreeLastPage(pBt); - if( idxNum==1 ){ - pCsr->pgno = sqlite3_value_int(argv[0]); - if( pCsr->pgno<1 || pCsr->pgno>pTab->nPage ){ + /* Default setting is no rows of result */ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + + if( idxNum & 2 ){ + const char *zSchema; + assert( argc>=1 ); + zSchema = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(db, zSchema); + if( pCsr->iDb<0 ) return SQLITE_OK; + }else{ + pCsr->iDb = 0; + } + pBt = db->aDb[pCsr->iDb].pBt; + if( pBt==0 ) return SQLITE_OK; + pCsr->pPager = sqlite3BtreePager(pBt); + pCsr->szPage = sqlite3BtreeGetPageSize(pBt); + pCsr->mxPgno = sqlite3BtreeLastPage(pBt); + if( idxNum & 1 ){ + assert( argc>(idxNum>>1) ); + pCsr->pgno = sqlite3_value_int(argv[idxNum>>1]); + if( pCsr->pgno<1 || pCsr->pgno>pCsr->mxPgno ){ pCsr->pgno = 1; pCsr->mxPgno = 0; }else{ pCsr->mxPgno = pCsr->pgno; } }else{ - pCsr->pgno = 1; - pCsr->mxPgno = pTab->nPage; + assert( pCsr->pgno==1 ); } + rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0); return rc; } @@ -212,7 +264,6 @@ static int dbpageColumn( int i ){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; - DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; int rc = SQLITE_OK; switch( i ){ case 0: { /* pgno */ @@ -221,9 +272,9 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pTab->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pTab->szPage, + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, SQLITE_TRANSIENT); } sqlite3PagerUnref(pDbPage); @@ -231,7 +282,7 @@ static int dbpageColumn( } default: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); - sqlite3_result_text(ctx, db->aDb[pTab->iDb].zDbSName, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, db->aDb[pCsr->iDb].zDbSName, -1, SQLITE_STATIC); break; } } @@ -255,33 +306,47 @@ static int dbpageUpdate( DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; + const char *zSchema; + int iDb; + Btree *pBt; + Pager *pPager; + int szPage; if( argc==1 ){ zErr = "cannot delete"; goto update_fail; } pgno = sqlite3_value_int(argv[0]); - if( pgno<1 || pgno>pTab->nPage ){ - zErr = "bad page number"; - goto update_fail; - } if( sqlite3_value_int(argv[1])!=pgno ){ zErr = "cannot insert"; goto update_fail; } + zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } + pBt = pTab->db->aDb[iDb].pBt; + if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){ + zErr = "bad page number"; + goto update_fail; + } + szPage = sqlite3BtreeGetPageSize(pBt); if( sqlite3_value_type(argv[3])!=SQLITE_BLOB - || sqlite3_value_bytes(argv[3])!=pTab->szPage + || sqlite3_value_bytes(argv[3])!=szPage ){ zErr = "bad page value"; goto update_fail; } - rc = sqlite3PagerGet(pTab->pPager, pgno, (DbPage**)&pDbPage, 0); + pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ memcpy(sqlite3PagerGetData(pDbPage), sqlite3_value_blob(argv[3]), - pTab->szPage); + szPage); } } sqlite3PagerUnref(pDbPage); @@ -293,6 +358,22 @@ update_fail: return SQLITE_ERROR; } +/* Since we do not know in advance which database files will be +** written by the sqlite_dbpage virtual table, start a write transaction +** on them all. +*/ +static int dbpageBegin(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3 *db = pTab->db; + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ) sqlite3BtreeBeginTrans(pBt, 1); + } + return SQLITE_OK; +} + + /* ** Invoke this routine to register the "dbpage" virtual table module */ @@ -312,7 +393,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageColumn, /* xColumn - read data */ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ - 0, /* xBegin */ + dbpageBegin, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ diff --git a/test/dbpage.test b/test/dbpage.test index a8ef8c0800..5a68d3caba 100644 --- a/test/dbpage.test +++ b/test/dbpage.test @@ -49,6 +49,13 @@ do_execsql_test 140 { do_execsql_test 150 { SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=0; } {} +do_execsql_test 160 { + ATTACH ':memory:' AS aux1; + PRAGMA aux1.page_size=4096; + CREATE TABLE aux1.t2(a,b,c); + INSERT INTO t2 VALUES(11,12,13); + SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('aux1'); +} {1 X'53514C6974' 2 X'0D00000001'} do_execsql_test 200 { CREATE TEMP TABLE saved_content(x); @@ -67,8 +74,23 @@ do_execsql_test 230 { do_catchsql_test 230 { PRAGMA integrity_check; } {0 ok} - - - +do_execsql_test 240 { + DELETE FROM saved_content; + INSERT INTO saved_content(x) + SELECT data FROM sqlite_dbpage WHERE schema='aux1' AND pgno=2; +} {} +do_execsql_test 241 { + UPDATE sqlite_dbpage SET data=zeroblob(4096) WHERE pgno=2 AND schema='aux1'; +} {} +do_catchsql_test 250 { + PRAGMA aux1.integrity_check; +} {1 {database disk image is malformed}} +do_execsql_test 260 { + UPDATE sqlite_dbpage SET data=(SELECT x FROM saved_content) + WHERE pgno=2 AND schema='aux1'; +} {} +do_catchsql_test 270 { + PRAGMA aux1.integrity_check; +} {0 ok} finish_test From b5039fb7d3944049da6c48e83a6d7685b5cd6da2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Oct 2017 23:28:13 +0000 Subject: [PATCH 045/156] Use extra locks to prevent a readonly_shm=1 process from connecting to a WAL-mode database if there are no writers. FossilOrigin-Name: 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 --- manifest | 17 +++++---- manifest.uuid | 2 +- src/os_unix.c | 96 ++++++++++++++++++++++++++++++------------------- src/sqlite.h.in | 1 + 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index a593818258..bf7be72e5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase. -D 2017-10-25T19:18:33.009 +C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters. +D 2017-10-25T23:28:13.771 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df +F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f -F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 +F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 @@ -1666,7 +1666,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 -R de06ad2d264283eba96aba95f64ee6f5 +P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 +R 7c17d54c5cff309a6f13f49dd620b485 +T *branch * readonly-wal-recovery +T *sym-readonly-wal-recovery * +T -sym-trunk * U drh -Z 3b4d77a61659a96790923a30bda00b59 +Z 26f9af4fe6668c97b92039b43092c035 diff --git a/manifest.uuid b/manifest.uuid index ce16a7caf0..44570bf71c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 \ No newline at end of file +35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c33d21fd9b..dadfaf9908 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4145,6 +4145,7 @@ struct unixShm { */ #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +#define UNIX_SHM_N_DMS 1000000000 /* Size of the DMS */ /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -4166,12 +4167,6 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); - /* Shared locks never span more than one byte */ - assert( n==1 || lockType!=F_RDLCK ); - - /* Locks are within range */ - assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); @@ -4186,36 +4181,44 @@ static int unixShmSystemLock( /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG - { u16 mask; - OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; + if( ofst=1 && n<=SQLITE_SHM_NLOCK ); + + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } } - }else{ - if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); - } - } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); } #endif @@ -4389,17 +4392,38 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. + /* Do not allow a read-only process to connect if there are no + ** writers, because a read-only process is unable to recover the + ** shm file following a system crash. */ rc = SQLITE_OK; + if( pShmNode->isReadonly ){ + if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){ + rc = SQLITE_CANTOPEN_DIRTYWAL; + } + } + + /* If we are able to grab the dead-man switch, that means this is the + ** first (write-enable) process to connect to the database. In that + ** case, truncate the shm file because the contents found on disk might + ** be invalid leftovers from a system crash. The shm will be rebuilt + */ if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } + + /* Acquires locks to tell other processes that a this process is + ** running and therefore the shm is valid they do not need to run + ** recovery. + */ if( rc==SQLITE_OK ){ + unsigned r; rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + sqlite3_randomness(sizeof(r), &r); + r = 1 + (r%(UNIX_SHM_N_DMS-1)); + (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1); } if( rc ) goto shm_open_err; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9ff366304f..0c0ed25b30 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -508,6 +508,7 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) From 4544e28425a02ee874e12e36ba7f0f9dfbf9c4b6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 26 Oct 2017 15:21:56 +0000 Subject: [PATCH 046/156] Fix a bug causing "make amalgmation-autoconf" to create an invalid package if either the current or parent directory contains a file named install.sh or install-sh. FossilOrigin-Name: e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d --- autoconf/configure.ac | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 5a607de054..c207bdf74d 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -12,6 +12,7 @@ AC_PREREQ(2.61) AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org) AC_CONFIG_SRCDIR([sqlite3.c]) +AC_CONFIG_AUX_DIR([.]) # Use automake. AM_INIT_AUTOMAKE([foreign]) diff --git a/manifest b/manifest index a593818258..03eafd6f57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase. -D 2017-10-25T19:18:33.009 +C Fix\sa\sbug\scausing\s"make\samalgmation-autoconf"\sto\screate\san\sinvalid\spackage\sif\neither\sthe\scurrent\sor\sparent\sdirectory\scontains\sa\sfile\snamed\sinstall.sh\sor\ninstall-sh. +D 2017-10-26T15:21:56.963 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -14,7 +14,7 @@ F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6 F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 +F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 -R de06ad2d264283eba96aba95f64ee6f5 -U drh -Z 3b4d77a61659a96790923a30bda00b59 +P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 +R b4e850a770e92ea272826b571ec0824a +U dan +Z d2098b60d98f2c55ca7e6fbf02761c40 diff --git a/manifest.uuid b/manifest.uuid index ce16a7caf0..8c0d2fd56c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 \ No newline at end of file +e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d \ No newline at end of file From 9181ae990a3b8623d27813209e6649bf76565545 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 26 Oct 2017 17:05:22 +0000 Subject: [PATCH 047/156] Instead of extra locks, use F_GETLK to ensure that readonly_shm clients cannot connect to a wal-mode database if there are no writers. FossilOrigin-Name: 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 --- manifest | 17 ++++---- manifest.uuid | 2 +- src/os_unix.c | 107 ++++++++++++++++++++++---------------------------- 3 files changed, 56 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index bf7be72e5e..c648e5a856 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters. -D 2017-10-25T23:28:13.771 +C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters. +D 2017-10-26T17:05:22.656 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb +F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1666,10 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 -R 7c17d54c5cff309a6f13f49dd620b485 -T *branch * readonly-wal-recovery -T *sym-readonly-wal-recovery * -T -sym-trunk * -U drh -Z 26f9af4fe6668c97b92039b43092c035 +P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 +R 87551e9df4f45e3704d812374bd0d192 +U dan +Z a113fb74c5589c985f30184211e03e42 diff --git a/manifest.uuid b/manifest.uuid index 44570bf71c..1815e9a333 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 \ No newline at end of file +5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index dadfaf9908..f689ff75fa 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4145,7 +4145,6 @@ struct unixShm { */ #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ -#define UNIX_SHM_N_DMS 1000000000 /* Size of the DMS */ /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -4167,6 +4166,12 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); + + /* Locks are within range */ + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); @@ -4181,44 +4186,36 @@ static int unixShmSystemLock( /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG - if( ofst=1 && n<=SQLITE_SHM_NLOCK ); - - OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; - } + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; }else{ - if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); - } + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); } #endif @@ -4392,38 +4389,30 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Do not allow a read-only process to connect if there are no - ** writers, because a read-only process is unable to recover the - ** shm file following a system crash. - */ + /* Check to see if another process is holding the dead-man switch. + ** For a readonly_shm client, if no other process holds the DMS lock, + ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned. + ** Or, for a read-write connection, if no other process holds a + ** DMS lock the file is truncated to zero bytes in size. */ rc = SQLITE_OK; if( pShmNode->isReadonly ){ - if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ rc = SQLITE_CANTOPEN_DIRTYWAL; } - } - - /* If we are able to grab the dead-man switch, that means this is the - ** first (write-enable) process to connect to the database. In that - ** case, truncate the shm file because the contents found on disk might - ** be invalid leftovers from a system crash. The shm will be rebuilt - */ - if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } - - /* Acquires locks to tell other processes that a this process is - ** running and therefore the shm is valid they do not need to run - ** recovery. - */ if( rc==SQLITE_OK ){ - unsigned r; rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - sqlite3_randomness(sizeof(r), &r); - r = 1 + (r%(UNIX_SHM_N_DMS-1)); - (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1); } if( rc ) goto shm_open_err; } From ab04eff809c0ee16a913e469ce51acdfcaf20c5d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 26 Oct 2017 17:34:50 +0000 Subject: [PATCH 048/156] Fix an error in the previous commit on this branch. FossilOrigin-Name: f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- test/walro.test | 13 +++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c648e5a856..ebf38789b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters. -D 2017-10-26T17:05:22.656 +C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. +D 2017-10-26T17:34:50.823 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545 +F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 +F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 -R 87551e9df4f45e3704d812374bd0d192 +P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 +R 49c1c99b22f8e84ff20a7afe5d1d547b U dan -Z a113fb74c5589c985f30184211e03e42 +Z 07348874af35473af5ead94c9a86c9df diff --git a/manifest.uuid b/manifest.uuid index 1815e9a333..3da837219d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 \ No newline at end of file +f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f689ff75fa..7720a255ac 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4401,7 +4401,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ lock.l_start = UNIX_SHM_DMS; lock.l_len = 1; lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) { + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { rc = SQLITE_IOERR_LOCK; }else if( lock.l_type==F_UNLCK ){ rc = SQLITE_CANTOPEN_DIRTYWAL; diff --git a/test/walro.test b/test/walro.test index f46e44d4cb..09a33bbd3e 100644 --- a/test/walro.test +++ b/test/walro.test @@ -101,10 +101,11 @@ do_multiclient_test tn { code1 { db close } list [file exists test.db-wal] [file exists test.db-shm] } {1 1} + do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } - sql1 { SELECT * FROM t1 } - } {a b c d e f g h i j} + list [catch { sql1 { SELECT * FROM t1 } } msg] $msg + } {1 {unable to open database file}} do_test 1.2.3 { code1 { db close } @@ -113,10 +114,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {1 {unable to open database file}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_CANTOPEN} do_test 1.2.5 { file attributes test.db-shm -permissions rw-r--r-- @@ -161,10 +162,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {1 {unable to open database file}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_CANTOPEN} #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent From 7ea11066b62c0dac40ed249d1eb64d6f2b9e01e3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 26 Oct 2017 18:43:19 +0000 Subject: [PATCH 049/156] Extra ".selecttrace" output following aggregate analysis. No changes to production builds. FossilOrigin-Name: 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 20 +++++++++++++++++++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 03eafd6f57..8c7e2d8801 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\scausing\s"make\samalgmation-autoconf"\sto\screate\san\sinvalid\spackage\sif\neither\sthe\scurrent\sor\sparent\sdirectory\scontains\sa\sfile\snamed\sinstall.sh\sor\ninstall-sh. -D 2017-10-26T15:21:56.963 +C Extra\s".selecttrace"\soutput\sfollowing\saggregate\sanalysis.\s\sNo\schanges\sto\nproduction\sbuilds. +D 2017-10-26T18:43:19.787 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -463,7 +463,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8 +F src/select.c d055dc3a428ab2dd94ffdbaef50d90b41579be8ffe258591615d3d83bd859f7b F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 -R b4e850a770e92ea272826b571ec0824a -U dan -Z d2098b60d98f2c55ca7e6fbf02761c40 +P e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d +R d61bdf0ef8749a81bf4548782da0df22 +U drh +Z eef6957fcaef94fa72ebc9ad261c03d1 diff --git a/manifest.uuid b/manifest.uuid index 8c0d2fd56c..0db876cf23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d \ No newline at end of file +6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0e2328120b..0f73c30f64 100644 --- a/src/select.c +++ b/src/select.c @@ -5659,6 +5659,24 @@ int sqlite3Select( } sAggInfo.mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + int ii; + SELECTTRACE(0x400,pParse,p,("After aggregate analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); + for(ii=0; ii Date: Thu, 26 Oct 2017 20:04:28 +0000 Subject: [PATCH 050/156] Enhance the min/max optimization so that it works with indexes on expressions. FossilOrigin-Name: 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833 --- manifest | 14 ++--- manifest.uuid | 2 +- src/select.c | 135 ++++++++++++++++++---------------------------- test/minmax2.test | 34 ++++++++++++ 4 files changed, 95 insertions(+), 90 deletions(-) diff --git a/manifest b/manifest index 8c7e2d8801..f1127b40df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\s".selecttrace"\soutput\sfollowing\saggregate\sanalysis.\s\sNo\schanges\sto\nproduction\sbuilds. -D 2017-10-26T18:43:19.787 +C Enhance\sthe\smin/max\soptimization\sso\sthat\sit\sworks\swith\sindexes\son\nexpressions. +D 2017-10-26T20:04:28.629 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -463,7 +463,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c d055dc3a428ab2dd94ffdbaef50d90b41579be8ffe258591615d3d83bd859f7b +F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1044,7 +1044,7 @@ F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41 -F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc +F test/minmax2.test dae92964ac87c1d2ef978c582e81a95e11c00f1cbef68980bfb2abaf10315063 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f F test/misc1.test 76737c259537586355f45e2a1e121b6e91b5476c4604ad5c53d1abfcb3acf786 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d -R d61bdf0ef8749a81bf4548782da0df22 +P 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76 +R c17176b8dc9ac053e5191ecbda77db6f U drh -Z eef6957fcaef94fa72ebc9ad261c03d1 +Z 1bd9fa3ff01b6d52accac3363bceb595 diff --git a/manifest.uuid b/manifest.uuid index 0db876cf23..41f1ba3965 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76 \ No newline at end of file +6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0f73c30f64..69ef06aeb5 100644 --- a/src/select.c +++ b/src/select.c @@ -3919,42 +3919,44 @@ static int pushDownWhereTerms( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Based on the contents of the AggInfo structure indicated by the first -** argument, this function checks if the following are true: +** The pFunc is the only aggregate function in the query. Check to see +** if the query is a candidate for the min/max optimization. ** -** * the query contains just a single aggregate function, -** * the aggregate function is either min() or max(), and -** * the argument to the aggregate function is a column value. +** If the query is a candidate for the min/max optimization, then set +** *ppMinMax to be an ORDER BY clause to be used for the optimization +** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on +** whether pFunc is a min() or max() function. ** -** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX -** is returned as appropriate. Also, *ppMinMax is set to point to the -** list of arguments passed to the aggregate before returning. +** If the query is not a candidate for the min/max optimization, return +** WHERE_ORDERBY_NORMAL (which must be zero). ** -** Or, if the conditions above are not met, *ppMinMax is set to 0 and -** WHERE_ORDERBY_NORMAL is returned. +** This routine must be called after aggregate functions have been +** located but before their arguments have been subjected to aggregate +** analysis. */ -static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ - int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ +static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + ExprList *pEList = pFunc->x.pList; /* Arguments to agg function */ + const char *zFunc; /* Name of aggregate function pFunc */ + ExprList *pOrderBy; + u8 sortOrder; - *ppMinMax = 0; - if( pAggInfo->nFunc==1 ){ - Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ - ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ - - assert( pExpr->op==TK_AGG_FUNCTION ); - if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ - const char *zFunc = pExpr->u.zToken; - if( sqlite3StrICmp(zFunc, "min")==0 ){ - eRet = WHERE_ORDERBY_MIN; - *ppMinMax = pEList; - }else if( sqlite3StrICmp(zFunc, "max")==0 ){ - eRet = WHERE_ORDERBY_MAX; - *ppMinMax = pEList; - } - } + assert( *ppMinMax==0 ); + assert( pFunc->op==TK_AGG_FUNCTION ); + if( pEList==0 || pEList->nExpr!=1 ) return eRet; + zFunc = pFunc->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + sortOrder = SQLITE_SO_ASC; + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + sortOrder = SQLITE_SO_DESC; + }else{ + return eRet; } - - assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); + assert( pOrderBy!=0 || db->mallocFailed ); + if( pOrderBy ) pOrderBy->a[0].sortOrder = sortOrder; return eRet; } @@ -5135,6 +5137,8 @@ int sqlite3Select( AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ + ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */ + u8 minMaxFlag; /* Flag for min/max queries */ #ifndef SQLITE_OMIT_EXPLAIN int iRestoreSelectId = pParse->iSelectId; @@ -5651,6 +5655,11 @@ int sqlite3Select( sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } sAggInfo.nAccumulator = sAggInfo.nColumn; + if( p->pGroupBy==0 && p->pHaving==0 && sAggInfo.nFunc==1 ){ + minMaxFlag = minMaxQuery(db, sAggInfo.aFunc[0].pExpr, &pMinMaxOrderBy); + }else{ + minMaxFlag = WHERE_ORDERBY_NORMAL; + } for(i=0; ipGroupBy==0 ); - assert( flag==0 ); - if( p->pHaving==0 ){ - flag = minMaxQuery(&sAggInfo, &pMinMax); - } - assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); - - if( flag ){ - pMinMax = sqlite3ExprListDup(db, pMinMax, 0); - pDel = pMinMax; - assert( db->mallocFailed || pMinMax!=0 ); - if( !db->mallocFailed ){ - pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; - pMinMax->a[0].pExpr->op = TK_COLUMN; - } - } - /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ + assert( p->pGroupBy==0 ); resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0); + + /* If this query is a candidate for the min/max optimization, then + ** minMaxFlag will have been previously set to either + ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will + ** be an appropriate ORDER BY expression for the optimization. + */ + assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); + assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); + + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, + 0, minMaxFlag, 0); if( pWInfo==0 ){ - sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", - (flag==WHERE_ORDERBY_MIN?"min":"max"))); + (minMaxFlag==WHERE_ORDERBY_MIN?"min":"max"))); } sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, &sAggInfo); @@ -6038,7 +6010,6 @@ int sqlite3Select( sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, -1, 0, 0, pDest, addrEnd, addrEnd); - sqlite3ExprListDelete(db, pDel); } sqlite3VdbeResolveLabel(v, addrEnd); @@ -6070,7 +6041,7 @@ int sqlite3Select( */ select_end: explainSetInteger(pParse->iSelectId, iRestoreSelectId); - + sqlite3ExprListDelete(db, pMinMaxOrderBy); sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); #if SELECTTRACE_ENABLED diff --git a/test/minmax2.test b/test/minmax2.test index da8fec30ce..b6114f2e51 100644 --- a/test/minmax2.test +++ b/test/minmax2.test @@ -383,5 +383,39 @@ do_test minmax2-10.12 { } } {{} {}} +# 2017-10-26. Extend the min/max optimization to indexes on expressions +# +do_execsql_test minmax2-11.100 { + CREATE TABLE t11(a,b,c); + INSERT INTO t11(a,b,c) VALUES(1,10,5),(2,8,11),(3,1,4),(4,20,1),(5,16,4); + CREATE INDEX t11bc ON t11(b+c); + SELECT max(b+c) FROM t11; +} {21} +do_execsql_test minmax2-11.110 { + SELECT a, max(b+c) FROM t11; +} {4 21} +do_test minmax2-11.111 { + db eval {SELECT max(b+c) FROM t11} + db status step +} {0} +do_test minmax2-11.112 { + db eval {SELECT max(c+b) FROM t11} + db status step +} {4} +do_execsql_test minmax2-11.120 { + SELECT a, min(b+c) FROM t11; +} {3 5} +do_test minmax2-11.121 { + db eval {SELECT min(b+c) FROM t11} + db status step +} {0} +do_test minmax2-11.122 { + db eval {SELECT min(c+b) FROM t11} + db status step +} {4} +do_execsql_test minmax2-11.130 { + INSERT INTO t11(a,b,c) VALUES(6,NULL,0),(7,0,NULL); + SELECT a, min(b+c) FROM t11; +} {3 5} finish_test From 6e61c5f895d77cba99c54c6b5b758f423e25c580 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Oct 2017 12:27:25 +0000 Subject: [PATCH 051/156] Additional test case demonstrating the ability to access structure elements using quoted names in the JSON1 extension. FossilOrigin-Name: 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json101.test | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f1127b40df..9f32f230d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\smin/max\soptimization\sso\sthat\sit\sworks\swith\sindexes\son\nexpressions. -D 2017-10-26T20:04:28.629 +C Additional\stest\scase\sdemonstrating\sthe\sability\sto\saccess\sstructure\selements\nusing\squoted\snames\sin\sthe\sJSON1\sextension. +D 2017-10-27T12:27:25.402 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -988,7 +988,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 2bd3703a8566e39f8efee8b83bb60aac363fa312e69c94b4024114e0fe77e380 +F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76 -R c17176b8dc9ac053e5191ecbda77db6f +P 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833 +R 8ebb970e40e0966c50eb4f87e0c6dbf4 U drh -Z 1bd9fa3ff01b6d52accac3363bceb595 +Z eb1c1a1b07b761c18b2113b5597a4235 diff --git a/manifest.uuid b/manifest.uuid index 41f1ba3965..f88c90a99a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833 \ No newline at end of file +8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 7b0292ad03..4bfcc2d80d 100644 --- a/test/json101.test +++ b/test/json101.test @@ -722,4 +722,44 @@ do_execsql_test json-11.3 { /* } */ } {0} +# 2017-10-27. Demonstrate the ability to access an element from +# a json structure even though the element name constains a "." +# character, by quoting the element name in the path. +# +do_execsql_test json-12.100 { + CREATE TABLE t12(x); + INSERT INTO t12(x) VALUES( + '{"settings": + {"layer2": + {"hapax.legomenon": + {"forceDisplay":true, + "transliterate":true, + "add.footnote":true, + "summary.report":true}, + "dis.legomenon": + {"forceDisplay":true, + "transliterate":false, + "add.footnote":false, + "summary.report":true}, + "tris.legomenon": + {"forceDisplay":true, + "transliterate":false, + "add.footnote":false, + "summary.report":false} + } + } + }'); +} {} +do_execsql_test json-12.110 { + SELECT json_remove(x, '$.settings.layer2."dis.legomenon".forceDisplay') + FROM t12; +} {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}} +do_execsql_test json-12.120 { + SELECT json_extract(x, '$.settings.layer2."tris.legomenon"."summary.report"') + FROM t12; +} {0} + + + + finish_test From b4bab90b31afa3c4646dbae5f650b2b17cb64a69 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 27 Oct 2017 17:09:44 +0000 Subject: [PATCH 052/156] Handle the Ctrl-C event for the shell on Win32. FossilOrigin-Name: c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9f32f230d3..2d13f91b8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Additional\stest\scase\sdemonstrating\sthe\sability\sto\saccess\sstructure\selements\nusing\squoted\snames\sin\sthe\sJSON1\sextension. -D 2017-10-27T12:27:25.402 +C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32. +D 2017-10-27T17:09:44.546 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -464,7 +464,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11 -F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f +F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833 -R 8ebb970e40e0966c50eb4f87e0c6dbf4 -U drh -Z eb1c1a1b07b761c18b2113b5597a4235 +P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f +R b04098ed5c4b7451669ce8c4fddf24e2 +U mistachkin +Z 04b0a33e7f8b0430861d645ce26e5f19 diff --git a/manifest.uuid b/manifest.uuid index f88c90a99a..478c761165 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f \ No newline at end of file +c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8f5ed59e93..e85ebd3c93 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1205,7 +1205,6 @@ static void output_csv(ShellState *p, const char *z, int bSep){ } } -#ifdef SIGINT /* ** This routine runs when the user presses Ctrl-C */ @@ -1215,6 +1214,20 @@ static void interrupt_handler(int NotUsed){ if( seenInterrupt>2 ) exit(1); if( globalDb ) sqlite3_interrupt(globalDb); } + +#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) +/* +** This routine runs for console events (e.g. Ctrl-C) on Win32 +*/ +static BOOL WINAPI ConsoleCtrlHandler( + DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ +){ + if( dwCtrlType==CTRL_C_EVENT ){ + interrupt_handler(0); + return TRUE; + } + return FALSE; +} #endif #ifndef SQLITE_OMIT_AUTHORIZATION @@ -6689,6 +6702,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ #ifdef SIGINT signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); #endif #ifdef SQLITE_SHELL_DBNAME_PROC From 0503f2acfa16288c28a3d13d87f06ef7c65da19d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 27 Oct 2017 18:24:11 +0000 Subject: [PATCH 053/156] Fix a Pager ref-count leak in the sqlite_dbpage virtual table. FossilOrigin-Name: c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/dbpage.c | 3 ++- test/dbpage.test | 8 ++++++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2d13f91b8d..7abf007c96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32. -D 2017-10-27T17:09:44.546 +C Fix\sa\sPager\sref-count\sleak\sin\sthe\ssqlite_dbpage\svirtual\stable. +D 2017-10-27T18:24:11.866 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -413,7 +413,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c 196a072dffda3ad50bb078da86bb39f1d07cabe4f88e2b12482f4f5ffce73352 +F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304 @@ -715,7 +715,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbpage.test ace7a8d2d66b0ad043dacc61d206991184c9916de6862bf4ff165a7906a624af +F test/dbpage.test dbf50a4d361f9e45a979432c727506065113124478a7d2db12074fa655e65d6c F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f -R b04098ed5c4b7451669ce8c4fddf24e2 -U mistachkin -Z 04b0a33e7f8b0430861d645ce26e5f19 +P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de +R f85ba3328211fce31f6fd8c8e66f1d27 +U drh +Z a44af1749e0d20716f81debef1ff6db8 diff --git a/manifest.uuid b/manifest.uuid index 478c761165..f8e9e5b80a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de \ No newline at end of file +c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 54fb5beaa7..f287e72229 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -220,7 +220,7 @@ static int dbpageFilter( ){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; - int rc = SQLITE_OK; + int rc; sqlite3 *db = pTab->db; Btree *pBt; @@ -254,6 +254,7 @@ static int dbpageFilter( }else{ assert( pCsr->pgno==1 ); } + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0); return rc; } diff --git a/test/dbpage.test b/test/dbpage.test index 5a68d3caba..754bc17926 100644 --- a/test/dbpage.test +++ b/test/dbpage.test @@ -56,6 +56,14 @@ do_execsql_test 160 { INSERT INTO t2 VALUES(11,12,13); SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('aux1'); } {1 X'53514C6974' 2 X'0D00000001'} +do_execsql_test 170 { + CREATE TABLE aux1.x3(x,y,z); + INSERT INTO x3(x,y,z) VALUES(1,'main',1),(2,'aux1',1); + SELECT pgno, schema, substr(data,1,6) + FROM sqlite_dbpage, x3 + WHERE sqlite_dbpage.schema=x3.y AND sqlite_dbpage.pgno=x3.z + ORDER BY x3.x; +} {1 main SQLite 1 aux1 SQLite} do_execsql_test 200 { CREATE TEMP TABLE saved_content(x); From bde13e26949ea15d639724a0d8d301ead3ab8005 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 27 Oct 2017 20:53:16 +0000 Subject: [PATCH 054/156] Add the start of the "incremental_index_check" virtual table in ext/repair/checkindex.c. For incremental verification of index contents. FossilOrigin-Name: d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 --- ext/repair/checkindex.c | 442 ++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- test/checkindex.test | 91 +++++++++ 4 files changed, 545 insertions(+), 7 deletions(-) create mode 100644 ext/repair/checkindex.c create mode 100644 test/checkindex.test diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c new file mode 100644 index 0000000000..a74d1d4b75 --- /dev/null +++ b/ext/repair/checkindex.c @@ -0,0 +1,442 @@ +/* +** 2017 October 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +typedef struct CidxTable CidxTable; +typedef struct CidxCursor CidxCursor; + +struct CidxTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; +}; + +struct CidxCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; +}; + +static void *cidxMalloc(int *pRc, int n){ + void *pRet = 0; + assert( n!=0 ); + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(n); + if( pRet ){ + memset(pRet, 0, n); + }else{ + *pRc = SQLITE_NOMEM; + } + } + return pRet; +} + +static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + assert( pCsr->base.pVtab->zErrMsg==0 ); + pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + +/* +** Connect to then incremental_index_check virtual table. +*/ +static int cidxConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; + CidxTable *pRet; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE xyz(" + " errmsg TEXT, current_key TEXT," + " index_name HIDDEN, after_key HIDDEN" + ")" + ); + pRet = cidxMalloc(&rc, sizeof(CidxTable)); + if( pRet ){ + pRet->db = db; + } + + *ppVtab = (sqlite3_vtab*)pRet; + return rc; +} + +/* +** Disconnect from or destroy an incremental_index_check virtual table. +*/ +static int cidxDisconnect(sqlite3_vtab *pVtab){ + CidxTable *pTab = (CidxTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex method. +*/ +static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ + int iIdxName = -1; + int iAfterKey = -1; + int i; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + + if( p->iColumn==2 ){ + iIdxName = i; + } + if( p->iColumn==3 ){ + iAfterKey = i; + } + } + + if( iIdxName<0 ){ + pInfo->estimatedCost = 1000000000.0; + }else{ + pInfo->aConstraintUsage[iIdxName].argvIndex = 1; + pInfo->aConstraintUsage[iIdxName].omit = 1; + if( iAfterKey<0 ){ + pInfo->estimatedCost = 1000000.0; + }else{ + pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; + pInfo->aConstraintUsage[iAfterKey].omit = 1; + pInfo->estimatedCost = 1000.0; + } + } + + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + CidxCursor *pRet; + int rc = SQLITE_OK; + + pRet = cidxMalloc(&rc, sizeof(CidxCursor)); + + *ppCursor = (sqlite3_vtab_cursor*)pRet; + return rc; +} + +/* +** Close a btreeinfo cursor. +*/ +static int cidxClose(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int cidxNext(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + int rc = sqlite3_step(pCsr->pStmt); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( rc!=SQLITE_OK ){ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); + } + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int cidxEof(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + return pCsr->pStmt==0; +} + +static sqlite3_stmt *cidxPrepare( + int *pRc, CidxCursor *pCsr, const char *zFmt, ... +){ + sqlite3_stmt *pRet = 0; + char *zSql; + va_list ap; /* ... printf arguments */ + va_start(ap, zFmt); + + zSql = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); + if( *pRc!=SQLITE_OK ){ + cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); + } + } + } + sqlite3_free(zSql); + va_end(ap); + + return pRet; +} + +static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +char *cidxStrdup(int *pRc, const char *zStr){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + int n = strlen(zStr); + zRet = cidxMalloc(pRc, n+1); + if( zRet ) memcpy(zRet, zStr, n+1); + } + return zRet; +} + +static int cidxLookupIndex( + CidxCursor *pCsr, /* Cursor object */ + const char *zIdx, /* Name of index to look up */ + char **pzTab, /* OUT: Table name */ + char **pzCurrentKey, /* OUT: Expression for current_key */ + char **pzOrderBy, /* OUT: ORDER BY expression list */ + char **pzSubWhere, /* OUT: sub-query WHERE clause */ + char **pzSubExpr /* OUT: sub-query WHERE clause */ +){ + int rc = SQLITE_OK; + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + + sqlite3_stmt *pFindTab = 0; + sqlite3_stmt *pGroup = 0; + + /* Find the table */ + pFindTab = cidxPrepare(&rc, pCsr, + "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", + zIdx + ); + if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + } + cidxFinalize(&rc, pFindTab); + if( rc==SQLITE_OK && zTab==0 ){ + rc = SQLITE_ERROR; + } + + pGroup = cidxPrepare(&rc, pCsr, + "SELECT group_concat(" + " coalesce(name, 'rowid'), '|| '','' ||'" + ") AS zCurrentKey," + " group_concat(" + " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," + " ', '" + ") AS zOrderBy," + " group_concat(" + " CASE WHEN key==1 THEN NULL ELSE " + " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " + " END," + " 'AND '" + ") AS zSubWhere," + " group_concat(" + " CASE WHEN key==0 THEN NULL ELSE " + " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " + " END," + " 'AND '" + ") AS zSubExpr " + " FROM pragma_index_xinfo(%Q);" + , zIdx, zIdx, zIdx + ); + if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){ + zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0)); + zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); + zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); + zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); + } + cidxFinalize(&rc, pGroup); + + if( rc!=SQLITE_OK ){ + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + }else{ + *pzTab = zTab; + *pzCurrentKey = zCurrentKey; + *pzOrderBy = zOrderBy; + *pzSubWhere = zSubWhere; + *pzSubExpr = zSubExpr; + } + + return rc; +} + +/* +** Position a cursor back to the beginning. +*/ +static int cidxFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + int rc = SQLITE_OK; + CidxCursor *pCsr = (CidxCursor*)pCursor; + const char *zIdxName = 0; + const char *zAfterKey = 0; + + if( argc>0 ){ + zIdxName = (const char*)sqlite3_value_text(argv[0]); + if( argc>1 ){ + zAfterKey = (const char*)sqlite3_value_text(argv[1]); + } + } + + if( zIdxName ){ + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + + rc = cidxLookupIndex(pCsr, zIdxName, + &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr + ); + pCsr->pStmt = cidxPrepare(&rc, pCsr, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", + zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + ); + + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + } + + if( pCsr->pStmt ){ + assert( rc==SQLITE_OK ); + rc = cidxNext(pCursor); + } + return rc; +} + +/* Return a column for the sqlite_btreeinfo table */ +static int cidxColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int iCol +){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + assert( iCol==0 || iCol==1 ); + if( iCol==0 ){ + const char *zVal = 0; + if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ + if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ + zVal = "row data mismatch"; + } + }else{ + zVal = "row missing"; + } + sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); + }else{ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Register the virtual table modules with the database handle passed +** as the only argument. +*/ +static int ciInit(sqlite3 *db){ + static sqlite3_module cidx_module = { + 0, /* iVersion */ + 0, /* xCreate */ + cidxConnect, /* xConnect */ + cidxBestIndex, /* xBestIndex */ + cidxDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + cidxOpen, /* xOpen - open a cursor */ + cidxClose, /* xClose - close a cursor */ + cidxFilter, /* xFilter - configure scan constraints */ + cidxNext, /* xNext - advance a cursor */ + cidxEof, /* xEof - check for end of scan */ + cidxColumn, /* xColumn - read data */ + cidxRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + }; + return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkindex_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return ciInit(db); +} diff --git a/manifest b/manifest index 2d13f91b8d..e0740cdd7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32. -D 2017-10-27T17:09:44.546 +C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents. +D 2017-10-27T20:53:16.451 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,6 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 +F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -653,6 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b +F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1666,7 +1668,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f -R b04098ed5c4b7451669ce8c4fddf24e2 -U mistachkin -Z 04b0a33e7f8b0430861d645ce26e5f19 +P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de +R 38fb5f31777282a52c65b4e266007341 +T *branch * checkindex +T *sym-checkindex * +T -sym-trunk * +U dan +Z 12607f1e180da6aa96255bb6397017d4 diff --git a/manifest.uuid b/manifest.uuid index 478c761165..9ee1c84b8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de \ No newline at end of file +d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test new file mode 100644 index 0000000000..841dca75a2 --- /dev/null +++ b/test/checkindex.test @@ -0,0 +1,91 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the checkindex extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix checkindex + +ifcapable !vtab||!compound { + finish_test + return +} + +if {[file exists ../checkindex.so]==0} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES('one', 2); + INSERT INTO t1 VALUES('two', 4); + INSERT INTO t1 VALUES('three', 6); + INSERT INTO t1 VALUES('four', 8); + INSERT INTO t1 VALUES('five', 10); +} + +db enable_load_extension 1 +do_execsql_test 1.1 { + SELECT load_extension('../checkindex.so'); +} {{}} + +do_execsql_test 1.2 { + SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); +} { + 1 five,5 + 1 four,4 + 1 one,1 + 1 three,3 + 1 two,2 +} + + +do_test 1.3 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt1(a, b)} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + execsql { + UPDATE xt1 SET a='six' WHERE rowid=3; + DELETE FROM xt1 WHERE rowid = 5; + } + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_execsql_test 1.4 { + SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); +} { + 0 five,5 + 1 four,4 + 1 one,1 + 0 three,3 + 1 two,2 +} +do_execsql_test 1.5 { + SELECT errmsg, current_key FROM incremental_index_check('i1'); +} { + {row missing} five,5 + {} four,4 + {} one,1 + {row data mismatch} three,3 + {} two,2 +} + + + +finish_test + From 6d6e76f75faaf018a03672794bc65805e6c1207e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Oct 2017 12:20:09 +0000 Subject: [PATCH 055/156] Add test cases from OSSFuzz to prevent a regression in co-routine processing. FossilOrigin-Name: 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/with1.test | 13 ++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7abf007c96..723da88b0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sPager\sref-count\sleak\sin\sthe\ssqlite_dbpage\svirtual\stable. -D 2017-10-27T18:24:11.866 +C Add\stest\scases\sfrom\sOSSFuzz\sto\sprevent\sa\sregression\sin\sco-routine\nprocessing. +D 2017-10-28T12:20:09.635 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -1557,7 +1557,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/with1.test 732e3ef398dcecb609839cd5ef0cb63beb2a9eff31420f3b745fc55b9e85b61e +F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87 F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de -R f85ba3328211fce31f6fd8c8e66f1d27 +P c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2 +R 49302fb645a8bd5bf1702bc6c1b70644 U drh -Z a44af1749e0d20716f81debef1ff6db8 +Z 9899cde33cecabbc451b9857a10233e0 diff --git a/manifest.uuid b/manifest.uuid index f8e9e5b80a..0d14844e0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2 \ No newline at end of file +689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2 \ No newline at end of file diff --git a/test/with1.test b/test/with1.test index 252a32bc69..92fbc58fcc 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1003,6 +1003,17 @@ do_execsql_test 19.1 { SELECT * FROM t1; } {0 0 0 {SCAN SUBQUERY 1} 0 1 1 {SCAN SUBQUERY 1}} - +# 2017-10-28. +# See check-in https://sqlite.org/src/info/0926df095faf72c2 +# Tried to optimize co-routine processing by changing a Copy opcode +# into SCopy. But OSSFuzz found two (similar) cases where that optimization +# does not work. +# +do_execsql_test 20.1 { + WITH c(i)AS(VALUES(9)UNION SELECT~i FROM c)SELECT max(5)>i fROM c; +} {0} +do_execsql_test 20.2 { + WITH c(i)AS(VALUES(5)UNIoN SELECT 0)SELECT min(1)-i fROM c; +} {1} finish_test From 19465e7380f725f9bada9c338e55d7e95bf46ef3 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Oct 2017 20:31:25 +0000 Subject: [PATCH 056/156] Fix "after" parameter handling in the incremental_index_check code. FossilOrigin-Name: c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 --- ext/repair/checkindex.c | 238 ++++++++++++++++++++++++++++++++++++++-- manifest | 17 ++- manifest.uuid | 2 +- test/checkindex.test | 175 +++++++++++++++++++++++++---- 4 files changed, 389 insertions(+), 43 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index a74d1d4b75..f86cfdae7a 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -45,6 +45,13 @@ struct CidxCursor { sqlite3_stmt *pStmt; }; +typedef struct CidxColumn CidxColumn; +struct CidxColumn { + char *zName; + char *zColl; + int bDesc; +}; + static void *cidxMalloc(int *pRc, int n){ void *pRet = 0; assert( n!=0 ); @@ -193,6 +200,23 @@ static int cidxEof(sqlite3_vtab_cursor *pCursor){ return pCsr->pStmt==0; } +static char *cidxMprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + va_end(ap); + return zRet; +} + static sqlite3_stmt *cidxPrepare( int *pRc, CidxCursor *pCsr, const char *zFmt, ... ){ @@ -237,6 +261,8 @@ char *cidxStrdup(int *pRc, const char *zStr){ static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ + int *pnCol, /* OUT: Number of columns in index */ + CidxColumn **paCol, /* OUT: Columns */ char **pzTab, /* OUT: Table name */ char **pzCurrentKey, /* OUT: Expression for current_key */ char **pzOrderBy, /* OUT: ORDER BY expression list */ @@ -249,6 +275,7 @@ static int cidxLookupIndex( char *zOrderBy = 0; char *zSubWhere = 0; char *zSubExpr = 0; + CidxColumn *aCol = 0; sqlite3_stmt *pFindTab = 0; sqlite3_stmt *pGroup = 0; @@ -268,7 +295,7 @@ static int cidxLookupIndex( pGroup = cidxPrepare(&rc, pCsr, "SELECT group_concat(" - " coalesce(name, 'rowid'), '|| '','' ||'" + " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" ") AS zCurrentKey," " group_concat(" " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," @@ -278,14 +305,15 @@ static int cidxLookupIndex( " CASE WHEN key==1 THEN NULL ELSE " " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " " END," - " 'AND '" + " ' AND '" ") AS zSubWhere," " group_concat(" " CASE WHEN key==0 THEN NULL ELSE " " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " " END," - " 'AND '" - ") AS zSubExpr " + " ' AND '" + ") AS zSubExpr," + " count(*) AS nCol" " FROM pragma_index_xinfo(%Q);" , zIdx, zIdx, zIdx ); @@ -294,8 +322,49 @@ static int cidxLookupIndex( zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); + *pnCol = sqlite3_column_int(pGroup, 4); } cidxFinalize(&rc, pGroup); + + pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nByte = 0; + int nCol = 0; + while( sqlite3_step(pGroup)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pGroup, 2); + const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + if( zName==0 ) zName = "rowid"; + nCol++; + nByte += strlen(zName)+1 + strlen(zColl)+1; + } + rc = sqlite3_reset(pGroup); + aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte); + + if( rc==SQLITE_OK ){ + int iCol = 0; + char *z = (char*)&aCol[nCol]; + while( sqlite3_step(pGroup)==SQLITE_ROW ){ + int nName, nColl; + const char *zName = (const char*)sqlite3_column_text(pGroup, 2); + const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + if( zName==0 ) zName = "rowid"; + + nName = strlen(zName); + nColl = strlen(zColl); + memcpy(z, zName, nName); + aCol[iCol].zName = z; + z += nName+1; + + memcpy(z, zColl, nColl); + aCol[iCol].zColl = z; + z += nColl+1; + + aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3); + iCol++; + } + } + cidxFinalize(&rc, pGroup); + } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); @@ -303,17 +372,128 @@ static int cidxLookupIndex( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(aCol); }else{ *pzTab = zTab; *pzCurrentKey = zCurrentKey; *pzOrderBy = zOrderBy; *pzSubWhere = zSubWhere; *pzSubExpr = zSubExpr; + *paCol = aCol; } return rc; } +static int cidxDecodeAfter( + CidxCursor *pCsr, + int nCol, + const char *zAfterKey, + char ***pazAfter +){ + char **azAfter; + int rc = SQLITE_OK; + int nAfterKey = strlen(zAfterKey); + + azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); + if( rc==SQLITE_OK ){ + int i; + char *zCopy = (char*)&azAfter[nCol]; + char *p = zCopy; + memcpy(zCopy, zAfterKey, nAfterKey+1); + for(i=0; i='0' && *p<='9') + || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' + ){ + p++; + } + } + + while( *p==' ' ) p++; + if( *p!=(i==(nCol-1) ? '\0' : ',') ){ + goto parse_error; + } + *p++ = '\0'; + } + } + + *pazAfter = azAfter; + return rc; + + parse_error: + sqlite3_free(azAfter); + *pazAfter = 0; + cidxCursorError(pCsr, "%s", "error parsing after value"); + return SQLITE_ERROR; +} + +static char *cidxWhere( + int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull +){ + char *zRet = 0; + const char *zSep = ""; + int i; + + for(i=0; i"), + azAfter[iGt] + ); + }else{ + zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName); + } + + return zRet; +} + +static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){ + int i; + char *zRet = 0; + const char *zSep = ""; + for(i=0; ipStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", - zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr ); + if( rc==SQLITE_OK && zAfterKey ){ + rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter); + } + + if( rc || zAfterKey==0 ){ + pCsr->pStmt = cidxPrepare(&rc, pCsr, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", + zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + ); + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + }else{ + char *zList = cidxColumnList(&rc, aCol, nCol); + const char *zSep = ""; + char *zSql; + int i; + + zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSubExpr, zTab, zSubWhere, zCurrentKey + ); + for(i=nCol-1; i>=0; i--){ + int j; + if( aCol[i].bDesc && azAfter[i]==0 ) continue; + for(j=0; j<2; j++){ + char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j); + zSql = cidxMprintf(&rc, + "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zList, zTab, zWhere, zOrderBy + ); + zSep = " UNION ALL "; + if( aCol[i].bDesc==0 ) break; + } + } + zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); + sqlite3_free(zList); + + /* printf("SQL: %s\n", zSql); */ + pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); + } + sqlite3_free(zTab); sqlite3_free(zCurrentKey); sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(aCol); + sqlite3_free(azAfter); } if( pCsr->pStmt ){ diff --git a/manifest b/manifest index e0740cdd7b..8f56822a50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents. -D 2017-10-27T20:53:16.451 +C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode. +D 2017-10-28T20:31:25.564 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1 +F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4 +F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,10 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de -R 38fb5f31777282a52c65b4e266007341 -T *branch * checkindex -T *sym-checkindex * -T -sym-trunk * +P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 +R 31f10dd93d0cbe363e20ad50a747a726 U dan -Z 12607f1e180da6aa96255bb6397017d4 +Z 019388109f5e5cd896dce5eabbc0252a diff --git a/manifest.uuid b/manifest.uuid index 9ee1c84b8f..45cec907d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 \ No newline at end of file +c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 841dca75a2..bafa42334c 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -34,6 +34,8 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES('three', 6); INSERT INTO t1 VALUES('four', 8); INSERT INTO t1 VALUES('five', 10); + + CREATE INDEX i2 ON t1(a DESC); } db enable_load_extension 1 @@ -41,18 +43,61 @@ do_execsql_test 1.1 { SELECT load_extension('../checkindex.so'); } {{}} +proc incr_index_check {idx nStep} { + set Q { + SELECT errmsg, current_key FROM incremental_index_check($idx, $after) + LIMIT $nStep + } + + set res [list] + while {1} { + unset -nocomplain current_key + set res1 [db eval $Q] + if {[llength $res1]==0} break + set res [concat $res $res1] + set after [lindex $res end] + } + + return $res +} + +proc do_index_check_test {tn idx res} { + uplevel [list do_execsql_test $tn.1 " + SELECT errmsg, current_key FROM incremental_index_check('$idx'); + " $res] + + uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] + #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] +} + do_execsql_test 1.2 { SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); } { - 1 five,5 - 1 four,4 - 1 one,1 - 1 three,3 - 1 two,2 + 1 'five',5 + 1 'four',4 + 1 'one',1 + 1 'three',3 + 1 'two',2 } +do_index_check_test 1.3 i1 { + {} 'five',5 + {} 'four',4 + {} 'one',1 + {} 'three',3 + {} 'two',2 +} -do_test 1.3 { +do_index_check_test 1.4 i2 { + {} 'two',2 + {} 'three',3 + {} 'one',1 + {} 'four',4 + {} 'five',5 +} + +do_test 1.5 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot db eval {CREATE TABLE xt1(a, b)} @@ -66,26 +111,110 @@ do_test 1.3 { sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 } {} -do_execsql_test 1.4 { - SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); -} { - 0 five,5 - 1 four,4 - 1 one,1 - 0 three,3 - 1 two,2 -} -do_execsql_test 1.5 { - SELECT errmsg, current_key FROM incremental_index_check('i1'); -} { - {row missing} five,5 - {} four,4 - {} one,1 - {row data mismatch} three,3 - {} two,2 +do_index_check_test 1.6 i1 { + {row missing} 'five',5 + {} 'four',4 + {} 'one',1 + {row data mismatch} 'three',3 + {} 'two',2 } +do_index_check_test 1.7 i2 { + {} 'two',2 + {row data mismatch} 'three',3 + {} 'one',1 + {} 'four',4 + {row missing} 'five',5 +} +#-------------------------------------------------------------------------- +do_execsql_test 2.0 { + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); + + INSERT INTO t2 VALUES(1, NULL, 1, 1); + INSERT INTO t2 VALUES(2, 1, NULL, 1); + INSERT INTO t2 VALUES(3, 1, 1, NULL); + + INSERT INTO t2 VALUES(4, 2, 2, 1); + INSERT INTO t2 VALUES(5, 2, 2, 2); + INSERT INTO t2 VALUES(6, 2, 2, 3); + + INSERT INTO t2 VALUES(7, 2, 2, 1); + INSERT INTO t2 VALUES(8, 2, 2, 2); + INSERT INTO t2 VALUES(9, 2, 2, 3); + + CREATE INDEX i3 ON t2(b, c, d); + CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); + CREATE INDEX i5 ON t2(d, c DESC, b); +} + +do_index_check_test 2.1 i3 { + {} NULL,1,1,1 + {} 1,NULL,1,2 + {} 1,1,NULL,3 + {} 2,2,1,4 + {} 2,2,1,7 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,3,6 + {} 2,2,3,9 +} + +do_index_check_test 2.2 i4 { + {} 2,2,3,6 + {} 2,2,3,9 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,1,4 + {} 2,2,1,7 + {} 1,1,NULL,3 + {} 1,NULL,1,2 + {} NULL,1,1,1 +} + +do_index_check_test 2.3 i5 { + {} NULL,1,1,3 + {} 1,2,2,4 + {} 1,2,2,7 + {} 1,1,NULL,1 + {} 1,NULL,1,2 + {} 2,2,2,5 + {} 2,2,2,8 + {} 3,2,2,6 + {} 3,2,2,9 +} + +do_execsql_test 3.0 { + + CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; + CREATE INDEX t3wxy ON t3(w, x, y); + CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); + + INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); + + INSERT INTO t3 VALUES('a', NULL, NULL, 4); + INSERT INTO t3 VALUES('a', NULL, NULL, 5); + INSERT INTO t3 VALUES('a', NULL, NULL, 6); + + INSERT INTO t3 VALUES('a', 'b', NULL, 7); + INSERT INTO t3 VALUES('a', 'b', NULL, 8); + INSERT INTO t3 VALUES('a', 'b', NULL, 9); + +} + +do_index_check_test 3.1 t3wxy { + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 +} +do_index_check_test 3.2 t3wxy2 { + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 +} finish_test From fca23557fe1d3d246de864b3ebffd186e098b884 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Oct 2017 20:51:54 +0000 Subject: [PATCH 057/156] Reactivate query flattening when the result set of the outer query has no function calls or subqueries. This is a partial reversal of check-in [c9104b59]. Co-routines are still preferred if the outer query has a complex result set, but for simple results sets, query flattening is used. Check-in [4464f40ccd7] is completely backed out due to this change. FossilOrigin-Name: d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/expr.c | 1 + src/select.c | 48 +++++++++++++++++++++++++++++------------------- src/sqliteInt.h | 9 ++++++--- 5 files changed, 46 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 723da88b0d..c2b9f80f14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfrom\sOSSFuzz\sto\sprevent\sa\sregression\sin\sco-routine\nprocessing. -D 2017-10-28T12:20:09.635 +C Reactivate\squery\sflattening\swhen\sthe\sresult\sset\sof\sthe\souter\squery\shas\nno\sfunction\scalls\sor\ssubqueries.\s\sThis\sis\sa\spartial\sreversal\sof\s\ncheck-in\s[c9104b59].\s\sCo-routines\sare\sstill\spreferred\sif\sthe\souter\nquery\shas\sa\scomplex\sresult\sset,\sbut\sfor\ssimple\sresults\ssets,\squery\sflattening\nis\sused.\s\sCheck-in\s[4464f40ccd7]\sis\scompletely\sbacked\nout\sdue\sto\sthis\schange. +D 2017-10-28T20:51:54.404 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -416,7 +416,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 -F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304 +F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6 @@ -463,12 +463,12 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11 +F src/select.c 36345c63153e3e3e8ba533c6d3be6201c317358e8c2b34c51400abdf192515ab F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 +F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1666,7 +1666,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2 -R 49302fb645a8bd5bf1702bc6c1b70644 +P 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2 +Q -4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0 +R 645c7260930835d4324fc70970813d97 U drh -Z 9899cde33cecabbc451b9857a10233e0 +Z 4445aa956123d14dc2e74ce6385c6a77 diff --git a/manifest.uuid b/manifest.uuid index 0d14844e0c..d5d218e0f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2 \ No newline at end of file +d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 31e9a93865..0a5dc913f6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -952,6 +952,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ return 0; } pNew->x.pList = pList; + ExprSetProperty(pNew, EP_HasFunc); assert( !ExprHasProperty(pNew, EP_xIsSelect) ); sqlite3ExprSetHeightAndFlags(pParse, pNew); return pNew; diff --git a/src/select.c b/src/select.c index 69ef06aeb5..f31dd850ab 100644 --- a/src/select.c +++ b/src/select.c @@ -3383,12 +3383,11 @@ static void substSelect( ** (19) If the subquery uses LIMIT then the outer query may not ** have a WHERE clause. ** -** (**) Subsumed into (17d3). Was: If the sub-query is a compound select, -** then it must not use an ORDER BY clause - Ticket #3773. Because -** of (17d3), then only way to have a compound subquery is if it is -** the only term in the FROM clause of the outer query. But if the -** only term in the FROM clause has an ORDER BY, then it will be -** implemented as a co-routine and the flattener will never be called. +** (20) If the sub-query is a compound select, then it must not use +** an ORDER BY clause. Ticket #3773. We could relax this constraint +** somewhat by saying that the terms of the ORDER BY clause must +** appear as unmodified result columns in the outer query. But we +** have other optimizations in mind to deal with that case. ** ** (21) If the subquery uses LIMIT then the outer query may not be ** DISTINCT. (See ticket [752e1646fc]). @@ -3522,6 +3521,9 @@ static int flattenSubquery( ** queries. */ if( pSub->pPrior ){ + if( pSub->pOrderBy ){ + return 0; /* Restriction (20) */ + } if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ return 0; /* (17d1), (17d2), or (17d3) */ } @@ -3556,15 +3558,6 @@ static int flattenSubquery( */ assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 ); - /* Ex-restriction (20): - ** A compound subquery must be the only term in the FROM clause of the - ** outer query by restriction (17d3). But if that term also has an - ** ORDER BY clause, then the subquery will be implemented by co-routine - ** and so the flattener will never be invoked. Hence, it is not possible - ** for the subquery to be a compound and have an ORDER BY clause. - */ - assert( pSub->pPrior==0 || pSub->pOrderBy==0 ); - /***** If we reach this point, flattening is permitted. *****/ SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", pSub->zSelName, pSub, iFrom)); @@ -4343,6 +4336,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3 *db = pParse->db; Expr *pE, *pRight, *pExpr; u16 selFlags = p->selFlags; + u32 elistFlags = 0; p->selFlags |= SF_Expanded; if( db->mallocFailed ){ @@ -4455,6 +4449,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; + elistFlags |= pE->flags; } if( knExpr ){ /* @@ -4470,6 +4465,7 @@ static int selectExpander(Walker *pWalker, Select *p){ for(k=0; knExpr; k++){ pE = a[k].pExpr; + elistFlags |= pE->flags; pRight = pE->pRight; assert( pE->op!=TK_DOT || pRight!=0 ); if( pE->op!=TK_ASTERISK @@ -4599,9 +4595,14 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ExprListDelete(db, pEList); p->pEList = pNew; } - if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many columns in result set"); - return WRC_Abort; + if( p->pEList ){ + if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; + } + if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){ + p->selFlags |= SF_ComplexResult; + } } return WRC_Continue; } @@ -5225,7 +5226,9 @@ int sqlite3Select( if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; assert( pSub->pGroupBy==0 ); - /* If the subquery contains an ORDER BY clause and if + /* If the outer query contains a "complex" result set (that is, + ** if the result set of the outer query uses functions or subqueries) + ** and if the subquery contains an ORDER BY clause and if ** it will be implemented as a co-routine, then do not flatten. This ** restriction allows SQL constructs like this: ** @@ -5234,9 +5237,16 @@ int sqlite3Select( ** ** The expensive_function() is only computed on the 10 rows that ** are output, rather than every row of the table. + ** + ** The requirement that the outer query have a complex result set + ** means that flattening does occur on simpler SQL constraints without + ** the expensive_function() like: + ** + ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10); */ if( pSub->pOrderBy!=0 && i==0 + && (p->selFlags & SF_ComplexResult)!=0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0cc435d7b7..378b085021 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2402,7 +2402,7 @@ struct Expr { */ #define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ - /* 0x000004 // available for use */ +#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ /* 0x000008 // available for use */ #define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ @@ -2426,9 +2426,10 @@ struct Expr { #define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ /* -** Combinations of two or more EP_* flags +** The EP_Propagate mask is a set of properties that automatically propagate +** upwards into parent nodes. */ -#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ +#define EP_Propagate (EP_Collate|EP_Subquery|EP_HasFunc) /* ** These macros can be used to test, set, or clear bits in the @@ -2708,6 +2709,7 @@ struct NameContext { #define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ #define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ +#define NC_Complex 0x2000 /* True if a function or subquery seen */ /* ** An instance of the following structure contains all information @@ -2778,6 +2780,7 @@ struct Select { #define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ #define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ +#define SF_ComplexResult 0x40000 /* Result set contains subquery or function */ /* From b68db3187b893b12c1c690711e9318c524818d35 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 28 Oct 2017 20:54:51 +0000 Subject: [PATCH 058/156] Increase the version number for the next release - which is still months away but there have been significant query planner enhancements since the previous release. FossilOrigin-Name: 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 15 +++++++-------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index 6075c9a9ff..a7e7070f80 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.21.0 +3.22.0 diff --git a/configure b/configure index 64b95e912c..9e7aa526a2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.21.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.22.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.21.0' -PACKAGE_STRING='sqlite 3.21.0' +PACKAGE_VERSION='3.22.0' +PACKAGE_STRING='sqlite 3.22.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1464,7 +1464,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.22.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1529,7 +1529,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.21.0:";; + short | recursive ) echo "Configuration of sqlite 3.22.0:";; esac cat <<\_ACEOF @@ -1654,7 +1654,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.21.0 +sqlite configure 3.22.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2073,7 +2073,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.21.0, which was +It was created by sqlite $as_me 3.22.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12167,7 +12167,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.21.0, which was +This file was extended by sqlite $as_me 3.22.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12233,7 +12233,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.21.0 +sqlite config.status 3.22.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index c2b9f80f14..f18ede0142 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Reactivate\squery\sflattening\swhen\sthe\sresult\sset\sof\sthe\souter\squery\shas\nno\sfunction\scalls\sor\ssubqueries.\s\sThis\sis\sa\spartial\sreversal\sof\s\ncheck-in\s[c9104b59].\s\sCo-routines\sare\sstill\spreferred\sif\sthe\souter\nquery\shas\sa\scomplex\sresult\sset,\sbut\sfor\ssimple\sresults\ssets,\squery\sflattening\nis\sused.\s\sCheck-in\s[4464f40ccd7]\sis\scompletely\sbacked\nout\sdue\sto\sthis\schange. -D 2017-10-28T20:51:54.404 +C Increase\sthe\sversion\snumber\sfor\sthe\snext\srelease\s-\swhich\sis\sstill\smonths\naway\sbut\sthere\shave\sbeen\ssignificant\squery\splanner\senhancements\ssince\sthe\nprevious\srelease. +D 2017-10-28T20:54:51.351 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd -F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 +F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -30,7 +30,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure e9dbb65b61c36bb9622225de254b768d4816749ff4cb4d71307bb067095aceec x +F configure bdc49e9f0b0ced903ebdb2850362dd3391eeb88585e0429d12b94928d2873b6b x F configure.ac 369ebae6c04d9d2de5064e21d300f2f42f2fbf13235cabff9d1a54f2b2c4d05d F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941 @@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2 -Q -4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0 -R 645c7260930835d4324fc70970813d97 +P d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f +R d5f45aed899d7f014af4553e25cf640f U drh -Z 4445aa956123d14dc2e74ce6385c6a77 +Z 66b45fa3f9b3a864ae6a46fe2fdaa343 diff --git a/manifest.uuid b/manifest.uuid index d5d218e0f2..0576f4ab05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f \ No newline at end of file +457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09 \ No newline at end of file From 6fc66179092c843b8f352579709417f6f5e5190d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 08:04:38 +0000 Subject: [PATCH 059/156] Fix an issue in incremental_index_check with indexes that use non-default collation sequences. FossilOrigin-Name: 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 --- ext/repair/checkindex.c | 5 +++-- manifest | 14 ++++++------- manifest.uuid | 2 +- test/checkindex.test | 44 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index f86cfdae7a..b70b57122c 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -298,7 +298,8 @@ static int cidxLookupIndex( " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" ") AS zCurrentKey," " group_concat(" - " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," + " coalesce(name, 'rowid') || ' COLLATE ' || coll " + " || CASE WHEN desc THEN ' DESC' ELSE '' END," " ', '" ") AS zOrderBy," " group_concat(" @@ -313,7 +314,7 @@ static int cidxLookupIndex( " END," " ' AND '" ") AS zSubExpr," - " count(*) AS nCol" + " count(*) AS nCol" " FROM pragma_index_xinfo(%Q);" , zIdx, zIdx, zIdx ); diff --git a/manifest b/manifest index 8f56822a50..1aefe38098 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode. -D 2017-10-28T20:31:25.564 +C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences. +D 2017-10-30T08:04:38.448 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4 +F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966 +F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 -R 31f10dd93d0cbe363e20ad50a747a726 +P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 +R 35b9a81e4b8b6b857354a831ff9c29c1 U dan -Z 019388109f5e5cd896dce5eabbc0252a +Z cd0262c8d2e550da2f2d47bdfb3d2467 diff --git a/manifest.uuid b/manifest.uuid index 45cec907d3..7ea946e219 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 \ No newline at end of file +3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index bafa42334c..706171ff19 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -185,6 +185,7 @@ do_index_check_test 2.3 i5 { {} 3,2,2,9 } +#-------------------------------------------------------------------------- do_execsql_test 3.0 { CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; @@ -216,5 +217,48 @@ do_index_check_test 3.2 t3wxy2 { {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 } +#-------------------------------------------------------------------------- +# Test with an index that uses non-default collation sequences. +# +do_execsql_test 4.0 { + CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); + INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); + INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); + INSERT INTO t4 VALUES(3, 'aab', 'ddd'); + INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); + + CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); +} + +do_index_check_test 4.1 t4cc { + {} 'aaa','bbb',1 + {} 'AAA','CCC',2 + {} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + +do_test 4.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + execsql { + UPDATE xt4 SET c1='hello' WHERE rowid=2; + DELETE FROM xt4 WHERE rowid = 3; + } + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_index_check_test 4.3 t4cc { + {} 'aaa','bbb',1 + {row data mismatch} 'AAA','CCC',2 + {row missing} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + + + finish_test From 75650d4f9453db917f8198a8d276ed96590ffb8a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 17:05:18 +0000 Subject: [PATCH 060/156] In checkindex.c, use C code instead of SQL/group_concat() to compose various SQL clauses. This is to make it easier to support indexes on expressions. FossilOrigin-Name: 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 --- ext/repair/checkindex.c | 255 +++++++++++++++++++++------------------- manifest | 14 +-- manifest.uuid | 2 +- test/checkindex.test | 5 +- 4 files changed, 144 insertions(+), 132 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index b70b57122c..0393358567 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -47,9 +47,15 @@ struct CidxCursor { typedef struct CidxColumn CidxColumn; struct CidxColumn { - char *zName; - char *zColl; - int bDesc; + char *zExpr; /* Text for indexed expression */ + int bDesc; /* True for DESC columns, otherwise false */ + int bKey; /* Part of index, not PK */ +}; + +typedef struct CidxIndex CidxIndex; +struct CidxIndex { + int nCol; /* Elements in aCol[] array */ + CidxColumn aCol[1]; /* Array of indexed columns */ }; static void *cidxMalloc(int *pRc, int n){ @@ -258,29 +264,30 @@ char *cidxStrdup(int *pRc, const char *zStr){ return zRet; } +static void cidxFreeIndex(CidxIndex *pIdx){ + if( pIdx ){ + int i; + for(i=0; inCol; i++){ + sqlite3_free(pIdx->aCol[i].zExpr); + } + sqlite3_free(pIdx); + } +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ - int *pnCol, /* OUT: Number of columns in index */ - CidxColumn **paCol, /* OUT: Columns */ - char **pzTab, /* OUT: Table name */ - char **pzCurrentKey, /* OUT: Expression for current_key */ - char **pzOrderBy, /* OUT: ORDER BY expression list */ - char **pzSubWhere, /* OUT: sub-query WHERE clause */ - char **pzSubExpr /* OUT: sub-query WHERE clause */ + CidxIndex **ppIdx, /* OUT: Description of columns */ + char **pzTab /* OUT: Table name */ ){ int rc = SQLITE_OK; char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - CidxColumn *aCol = 0; + CidxIndex *pIdx = 0; sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pGroup = 0; + sqlite3_stmt *pInfo = 0; - /* Find the table */ + /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", zIdx @@ -293,94 +300,35 @@ static int cidxLookupIndex( rc = SQLITE_ERROR; } - pGroup = cidxPrepare(&rc, pCsr, - "SELECT group_concat(" - " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" - ") AS zCurrentKey," - " group_concat(" - " coalesce(name, 'rowid') || ' COLLATE ' || coll " - " || CASE WHEN desc THEN ' DESC' ELSE '' END," - " ', '" - ") AS zOrderBy," - " group_concat(" - " CASE WHEN key==1 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubWhere," - " group_concat(" - " CASE WHEN key==0 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubExpr," - " count(*) AS nCol" - " FROM pragma_index_xinfo(%Q);" - , zIdx, zIdx, zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){ - zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0)); - zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); - zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); - zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); - *pnCol = sqlite3_column_int(pGroup, 4); - } - cidxFinalize(&rc, pGroup); - - pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); if( rc==SQLITE_OK ){ - int nByte = 0; - int nCol = 0; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; if( zName==0 ) zName = "rowid"; - nCol++; - nByte += strlen(zName)+1 + strlen(zColl)+1; - } - rc = sqlite3_reset(pGroup); - aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte); - - if( rc==SQLITE_OK ){ - int iCol = 0; - char *z = (char*)&aCol[nCol]; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - int nName, nColl; - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); - if( zName==0 ) zName = "rowid"; - - nName = strlen(zName); - nColl = strlen(zColl); - memcpy(z, zName, nName); - aCol[iCol].zName = z; - z += nName+1; - - memcpy(z, zColl, nColl); - aCol[iCol].zColl = z; - z += nColl+1; - - aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3); - iCol++; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); } + p = &pIdx->aCol[iCol++]; + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + pIdx->nCol = iCol; } - cidxFinalize(&rc, pGroup); + cidxFinalize(&rc, pInfo); } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); }else{ *pzTab = zTab; - *pzCurrentKey = zCurrentKey; - *pzOrderBy = zOrderBy; - *pzSubWhere = zSubWhere; - *pzSubExpr = zSubExpr; - *paCol = aCol; + *ppIdx = pIdx; } return rc; @@ -463,35 +411,91 @@ static char *cidxWhere( int i; for(i=0; i"), + zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, + zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName); + zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); } return zRet; } -static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){ - int i; +#define CIDX_CLIST_ALL 0 +#define CIDX_CLIST_ORDERBY 1 +#define CIDX_CLIST_CURRENT_KEY 2 +#define CIDX_CLIST_SUBWHERE 3 +#define CIDX_CLIST_SUBEXPR 4 + +/* +** This function returns various strings based on the contents of the +** CidxIndex structure and the eType parameter. +*/ +static char *cidxColumnList( + int *pRc, /* IN/OUT: Error code */ + const char *zIdx, + CidxIndex *pIdx, /* Indexed columns */ + int eType /* True to include ASC/DESC */ +){ char *zRet = 0; - const char *zSep = ""; - for(i=0; inCol; i++){ + CidxColumn *p = &pIdx->aCol[i]; + assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); + switch( eType ){ + + case CIDX_CLIST_ORDERBY: + zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zSep = ","; + break; + + case CIDX_CLIST_CURRENT_KEY: + zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zSep = "||','||"; + break; + + case CIDX_CLIST_SUBWHERE: + if( p->bKey==0 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + case CIDX_CLIST_SUBEXPR: + if( p->bKey==1 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + default: + assert( eType==CIDX_CLIST_ALL ); + zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); + zSep = ","; + break; + } + } } + return zRet; } @@ -516,21 +520,26 @@ static int cidxFilter( } if( zIdxName ){ - int nCol = 0; char *zTab = 0; char *zCurrentKey = 0; char *zOrderBy = 0; char *zSubWhere = 0; char *zSubExpr = 0; - char **azAfter = 0; - CidxColumn *aCol = 0; + char *zSrcList = 0; - rc = cidxLookupIndex(pCsr, zIdxName, - &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr - ); + char **azAfter = 0; + CidxIndex *pIdx = 0; + + rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); + + zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); + zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); + zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); + zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); + /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter); + rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); } if( rc || zAfterKey==0 ){ @@ -538,9 +547,9 @@ static int cidxFilter( "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, aCol, nCol); + char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; @@ -548,17 +557,17 @@ static int cidxFilter( zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); - for(i=nCol-1; i>=0; i--){ + for(i=pIdx->nCol-1; i>=0; i--){ int j; - if( aCol[i].bDesc && azAfter[i]==0 ) continue; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j); + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); zSql = cidxMprintf(&rc, "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", zSql, zSep, zList, zTab, zWhere, zOrderBy ); zSep = " UNION ALL "; - if( aCol[i].bDesc==0 ) break; + if( pIdx->aCol[i].bDesc==0 ) break; } } zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); @@ -573,7 +582,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); sqlite3_free(azAfter); } @@ -584,7 +593,9 @@ static int cidxFilter( return rc; } -/* Return a column for the sqlite_btreeinfo table */ +/* +** Return a column value. +*/ static int cidxColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, diff --git a/manifest b/manifest index 1aefe38098..48fe0a91a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences. -D 2017-10-30T08:04:38.448 +C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. +D 2017-10-30T17:05:18.290 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0 +F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2 +F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 -R 35b9a81e4b8b6b857354a831ff9c29c1 +P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 +R 45b731228881e6db68d5d1d151d2789f U dan -Z cd0262c8d2e550da2f2d47bdfb3d2467 +Z 8f0b667700c59bebe92a45558d3d25e5 diff --git a/manifest.uuid b/manifest.uuid index 7ea946e219..5d8e25d480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 \ No newline at end of file +940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 706171ff19..6648187559 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -67,10 +67,11 @@ proc do_index_check_test {tn idx res} { " $res] uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] + uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] } + do_execsql_test 1.2 { SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); } { From eba21f9eb246db1e4e723ef9665c5bc5b6ec7749 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Oct 2017 18:49:11 +0000 Subject: [PATCH 061/156] Improve the performance of the LIKE operator by using strcspn() to aid wildcard matching. FossilOrigin-Name: 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 16 ++++++++++------ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f18ede0142..e88bdbcfad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sfor\sthe\snext\srelease\s-\swhich\sis\sstill\smonths\naway\sbut\sthere\shave\sbeen\ssignificant\squery\splanner\senhancements\ssince\sthe\nprevious\srelease. -D 2017-10-28T20:54:51.351 +C Improve\sthe\sperformance\sof\sthe\sLIKE\soperator\sby\susing\sstrcspn()\sto\said\nwildcard\smatching. +D 2017-10-30T18:49:11.595 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -419,7 +419,7 @@ F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 -F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6 +F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f -R d5f45aed899d7f014af4553e25cf640f +P 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09 +R b136c0336cd9a2670f5306d89a5f1d34 U drh -Z 66b45fa3f9b3a864ae6a46fe2fdaa343 +Z 1ff3723c7287a54f5348d686239a8395 diff --git a/manifest.uuid b/manifest.uuid index 0576f4ab05..e6859f5555 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09 \ No newline at end of file +37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4 \ No newline at end of file diff --git a/src/func.c b/src/func.c index b46578a407..7528fa8b4d 100644 --- a/src/func.c +++ b/src/func.c @@ -698,16 +698,20 @@ static int patternCompare( ** c or cx. */ if( c<=0x80 ){ - u32 cx; + char zStop[3]; int bMatch; if( noCase ){ - cx = sqlite3Toupper(c); - c = sqlite3Tolower(c); + zStop[0] = sqlite3Toupper(c); + zStop[1] = sqlite3Tolower(c); + zStop[2] = 0; }else{ - cx = c; + zStop[0] = c; + zStop[1] = 0; } - while( (c2 = *(zString++))!=0 ){ - if( c2!=c && c2!=cx ) continue; + while(1){ + zString += strcspn((const char*)zString, zStop); + if( zString[0]==0 ) break; + zString++; bMatch = patternCompare(zPattern,zString,pInfo,matchOther); if( bMatch!=SQLITE_NOMATCH ) return bMatch; } From e7a579d27d12ce936d5a536dffd7453bd0503235 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 19:38:41 +0000 Subject: [PATCH 062/156] Add support for indexes on expressions to incremental_index_check. FossilOrigin-Name: 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 --- ext/repair/checkindex.c | 215 +++++++++++++++++++++++++++++++--------- manifest | 14 +-- manifest.uuid | 2 +- test/checkindex.test | 58 +++++++++++ 4 files changed, 232 insertions(+), 57 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 0393358567..fc91b74244 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -274,6 +274,110 @@ static void cidxFreeIndex(CidxIndex *pIdx){ } } +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +static int cidxFindNext(const char *zIn, const char **pzOut){ + const char *z = zIn; + + while( 1 ){ + *pzOut = z; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + default: + z++; + } + } + + assert( 0 ); + return -1; +} + +static int cidx_isspace(char c){ + return c==' ' || c=='\t' || c=='\r' || c=='\n'; +} + +static int cidx_isident(char c){ + return c<0 + || (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ + const char *z = zSql; + const char *z1; + int e; + int rc = SQLITE_OK; + int nParen = 1; + CidxColumn *pCol = pIdx->aCol; + + e = cidxFindNext(z, &z); + if( e!=CIDX_PARSE_OPEN ) goto parse_error; + z1 = z+1; + z++; + while( nParen>0 ){ + e = cidxFindNext(z, &z); + if( e==CIDX_PARSE_EOF ) goto parse_error; + if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ + const char *z2 = z; + if( pCol->zExpr ) goto parse_error; + + while( cidx_isspace(z[-1]) ) z--; + if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + + while( cidx_isspace(z1[0]) ) z1++; + pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); + pCol++; + z = z1 = z2+1; + } + if( e==CIDX_PARSE_OPEN ) nParen++; + if( e==CIDX_PARSE_CLOSE ) nParen--; + z++; + } + + return rc; + + parse_error: + cidxCursorError(pCsr, "Parse error in: %s", zSql); + return SQLITE_ERROR; +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ @@ -289,39 +393,50 @@ static int cidxLookupIndex( /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", + "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'", zIdx ); if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; + if( zName==0 ) zName = "rowid"; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); + nAlloc += 8; + } + p = &pIdx->aCol[iCol++]; + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + if( zSql==0 || p->bKey==0 ){ + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + }else{ + p->zExpr = 0; + } + pIdx->nCol = iCol; + } + cidxFinalize(&rc, pInfo); + } + + if( rc==SQLITE_OK && zSql ){ + rc = cidxParseSQL(pCsr, pIdx, zSql); + } } + cidxFinalize(&rc, pFindTab); if( rc==SQLITE_OK && zTab==0 ){ rc = SQLITE_ERROR; } - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - } - p = &pIdx->aCol[iCol++]; - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - pIdx->nCol = iCol; - } - cidxFinalize(&rc, pInfo); - } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); @@ -411,22 +526,22 @@ static char *cidxWhere( int i; for(i=0; i"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); + zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); } return zRet; @@ -450,7 +565,7 @@ static char *cidxColumnList( ){ char *zRet = 0; if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {" ASC", " DESC"}; + const char *aDir[2] = {"", " DESC"}; int i; const char *zSep = ""; @@ -460,19 +575,19 @@ static char *cidxColumnList( switch( eType ){ case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); zSep = ","; break; case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); zSep = "||','||"; break; case CIDX_CLIST_SUBWHERE: if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -480,8 +595,8 @@ static char *cidxColumnList( case CIDX_CLIST_SUBEXPR: if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -489,8 +604,8 @@ static char *cidxColumnList( default: assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); - zSep = ","; + zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); + zSep = ", "; break; } } @@ -536,7 +651,7 @@ static int cidxFilter( zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ + zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); if( rc==SQLITE_OK && zAfterKey ){ rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); @@ -544,17 +659,19 @@ static int cidxFilter( if( rc || zAfterKey==0 ){ pCsr->pStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", - zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; - zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); for(i=pIdx->nCol-1; i>=0; i--){ @@ -562,16 +679,15 @@ static int cidxFilter( if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, - "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", - zSql, zSep, zList, zTab, zWhere, zOrderBy - ); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zSrcList, zTab, zWhere, zOrderBy + ); zSep = " UNION ALL "; if( pIdx->aCol[i].bDesc==0 ) break; } } - zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); - sqlite3_free(zList); + zSql = cidxMprintf(&rc, "%z) AS i", zSql); /* printf("SQL: %s\n", zSql); */ pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); @@ -582,6 +698,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(zSrcList); cidxFreeIndex(pIdx); sqlite3_free(azAfter); } diff --git a/manifest b/manifest index 48fe0a91a2..59ee792f38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. -D 2017-10-30T17:05:18.290 +C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check. +D 2017-10-30T19:38:41.046 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 +F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 +F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 -R 45b731228881e6db68d5d1d151d2789f +P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 +R 32cef05dfb77ed0c73cb51913be9d3e3 U dan -Z 8f0b667700c59bebe92a45558d3d25e5 +Z 4d91b1f15edf1ab6cd873cde5a983874 diff --git a/manifest.uuid b/manifest.uuid index 5d8e25d480..f363d67abb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file +8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 6648187559..a91455558f 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -259,6 +259,64 @@ do_index_check_test 4.3 t4cc { {} 'AAB','EEE',4 } +#-------------------------------------------------------------------------- +# Test an index on an expression. +# +do_execsql_test 5.0 { + CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); + INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); + INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); + INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); + INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); + INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); + + CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); + CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); +} + +do_index_check_test 5.1.1 t5x { + {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 +} + +do_index_check_test 5.1.2 t5y { + {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 +} + +do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { + {} {'{"w":4, "z":4}',4} + {} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} + +do_test 5.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + execsql { + UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; + DELETE FROM xt5 WHERE rowid = 4; + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_index_check_test 5.3.1 t5x { + {row missing} NULL,4 + {row data mismatch} 1,1 + {} 2,2 + {} 3,3 + {} 5,5 +} + +do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { + {row missing} {'{"w":4, "z":4}',4} + {row data mismatch} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} finish_test From 07dae088b9723550a91d492d92212f3afed89b5d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Oct 2017 20:44:36 +0000 Subject: [PATCH 063/156] Clarify some comments describing the WAL index file. No changes to code. FossilOrigin-Name: 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e88bdbcfad..0913ececd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\sLIKE\soperator\sby\susing\sstrcspn()\sto\said\nwildcard\smatching. -D 2017-10-30T18:49:11.595 +C Clarify\ssome\scomments\sdescribing\sthe\sWAL\sindex\sfile.\s\sNo\schanges\sto\scode. +D 2017-10-30T20:44:36.285 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a +F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09 -R b136c0336cd9a2670f5306d89a5f1d34 +P 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4 +R b73d22fe388dfec64db639a4eb4f982f U drh -Z 1ff3723c7287a54f5348d686239a8395 +Z b8d196e6ebc579e3254aa8271770e0e4 diff --git a/manifest.uuid b/manifest.uuid index e6859f5555..7d46081730 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4 \ No newline at end of file +3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 9930b84421..0785dd3b00 100644 --- a/src/wal.c +++ b/src/wal.c @@ -132,6 +132,10 @@ ** on a network filesystem. All users of the database must be able to ** share memory. ** +** In the default unix and windows implementation, the wal-index is a mmapped +** file whose name is the database name with a "-shm" suffix added. For that +** reason, the wal-index is sometimes called the "shm" file. +** ** The wal-index is transient. After a crash, the wal-index can (and should ** be) reconstructed from the original WAL file. In fact, the VFS is required ** to either truncate or zero the header of the wal-index when the last @@ -271,9 +275,18 @@ int sqlite3WalTrace = 0; #define WALINDEX_MAX_VERSION 3007000 /* -** Indices of various locking bytes. WAL_NREADER is the number +** Index numbers for various locking bytes. WAL_NREADER is the number ** of available reader locks and should be at least 3. The default ** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. +** +** Technically, the various VFSes are free to implement these locks however +** they see fit. However, compatibility is encouraged so that VFSes can +** interoperate. The standard implemention used on both unix and windows +** is for the index number to indicate a byte offset into the +** WalCkptInfo.aLock[] array in the wal-index header. In other words, all +** locks are on the shm file. The WALINDEX_LOCK_OFFSET constant (which +** should be 120) is the location in the shm file for the first locking +** byte. */ #define WAL_WRITE_LOCK 0 #define WAL_ALL_BUT_WRITE 1 @@ -397,7 +410,6 @@ struct WalCkptInfo { #define WAL_FRAME_HDRSIZE 24 /* Size of write ahead log header, including checksum. */ -/* #define WAL_HDRSIZE 24 */ #define WAL_HDRSIZE 32 /* WAL magic value. Either this value, or the same value with the least From 666fb6914d659cf9e997dfa5d97f3124cdad39a8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Oct 2017 23:25:06 +0000 Subject: [PATCH 064/156] Add the showshm utility program for printing out the shm header in a human-readable format. FossilOrigin-Name: f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d --- Makefile.in | 3 + Makefile.msc | 3 + main.mk | 3 + manifest | 17 +++--- manifest.uuid | 2 +- tool/showshm.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 tool/showshm.c diff --git a/Makefile.in b/Makefile.in index 5f6be57c37..d03f009b79 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1203,6 +1203,9 @@ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) +showshm$(TEXE): $(TOP)/tool/showshm.c + $(LTLINK) -o $@ $(TOP)/tool/showshm.c + changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 2804c25ab4..fb6f61979e 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2224,6 +2224,9 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +showshm.exe: $(TOP)\tool\showshm.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) + changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ diff --git a/main.mk b/main.mk index c7711fb6e9..346dd8feb0 100644 --- a/main.mk +++ b/main.mk @@ -908,6 +908,9 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) +showshm$(EXE): $(TOP)/tool/showshm.c + $(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c + changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index 0913ececd3..362607f4d0 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Clarify\ssome\scomments\sdescribing\sthe\sWAL\sindex\sfile.\s\sNo\schanges\sto\scode. -D 2017-10-30T20:44:36.285 -F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 +C Add\sthe\sshowshm\sutility\sprogram\sfor\sprinting\sout\sthe\sshm\sheader\sin\sa\nhuman-readable\sformat. +D 2017-10-30T23:25:06.424 +F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 +F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -386,7 +386,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8 +F main.mk a6e3fa3c4d145189a67ad89e3f0d6541e8c085f5c34a21df0b1c870eb3aeb6ce F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1622,6 +1622,7 @@ F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 +F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -1666,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4 -R b73d22fe388dfec64db639a4eb4f982f +P 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7 +R 5511878351634578cf57017f9253148a U drh -Z b8d196e6ebc579e3254aa8271770e0e4 +Z 5f57a6784d89256cd72e7dad10ab878c diff --git a/manifest.uuid b/manifest.uuid index 7d46081730..a5b1cdf01a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7 \ No newline at end of file +f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d \ No newline at end of file diff --git a/tool/showshm.c b/tool/showshm.c new file mode 100644 index 0000000000..03e0fc460c --- /dev/null +++ b/tool/showshm.c @@ -0,0 +1,158 @@ +/* +** A utility for printing content from the wal-index or "shm" file. +*/ +#include +#include +#include +#include +#include +#include + +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISPRINT(X) isprint((unsigned char)(X)) + +#if !defined(_MSC_VER) +#include +#include +#else +#include +#endif + +#include +#include + +static int fd = -1; /* The open SHM file */ + +/* Report an out-of-memory error and die. +*/ +static void out_of_memory(void){ + fprintf(stderr,"Out of memory...\n"); + exit(1); +} + +/* +** Read content from the file. +** +** Space to hold the content is obtained from malloc() and needs to be +** freed by the caller. +*/ +static unsigned char *getContent(int ofst, int nByte){ + unsigned char *aData; + aData = malloc(nByte); + if( aData==0 ) out_of_memory(); + lseek(fd, ofst, SEEK_SET); + read(fd, aData, nByte); + return aData; +} + +/* +** Flags values +*/ +#define FG_HEX 1 /* Show as hex */ +#define FG_NBO 2 /* Native byte order */ +#define FG_PGSZ 4 /* Show as page-size */ + +/* Print a line of decode output showing a 4-byte integer. +*/ +static void print_decode_line( + unsigned char *aData, /* Content being decoded */ + int ofst, int nByte, /* Start and size of decode */ + unsigned flg, /* Display flags */ + const char *zMsg /* Message to append */ +){ + int i, j; + int val = aData[ofst]; + char zBuf[100]; + sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); + i = (int)strlen(zBuf); + for(j=1; j<4; j++){ + if( j>=nByte ){ + sprintf(&zBuf[i], " "); + }else{ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + val = val*256 + aData[ofst+j]; + } + i += (int)strlen(&zBuf[i]); + } + if( nByte==8 ){ + for(j=4; j<8; j++){ + sprintf(&zBuf[i], " %02x", aData[ofst+j]); + i += (int)strlen(&zBuf[i]); + } + } + if( flg & FG_NBO ){ + assert( nByte==4 ); + memcpy(&val, aData+ofst, 4); + } + sprintf(&zBuf[i], " "); + i += 12; + if( flg & FG_PGSZ ){ + unsigned short sz; + memcpy(&sz, aData+ofst, 2); + sprintf(&zBuf[i], " %9d", sz==1 ? 65536 : sz); + }else if( flg & FG_HEX ){ + sprintf(&zBuf[i], " 0x%08x", val); + }else if( nByte<8 ){ + sprintf(&zBuf[i], " %9d", val); + } + printf("%s %s\n", zBuf, zMsg); +} + +/* +** Print an instance of the WalIndexHdr object. ix is either 0 or 1 +** to select which header to print. +*/ +static void print_index_hdr(unsigned char *aData, int ix){ + int i; + assert( ix==0 || ix==1 ); + i = ix ? 48 : 0; + print_decode_line(aData, 0+i, 4, FG_NBO, "Wal-index version"); + print_decode_line(aData, 4+i, 4, 0, "unused padding"); + print_decode_line(aData, 8+i, 4, FG_NBO, "transaction counter"); + print_decode_line(aData,12+i, 1, 0, "1 when initialized"); + print_decode_line(aData,13+i, 1, 0, "true if WAL cksums are bigendian"); + print_decode_line(aData,14+i, 2, FG_PGSZ, "database page size"); + print_decode_line(aData,16+i, 4, FG_NBO, "mxFrame"); + print_decode_line(aData,20+i, 4, FG_NBO, "Size of database in pages"); + print_decode_line(aData,24+i, 8, 0, "Cksum of last frame in -wal"); + print_decode_line(aData,32+i, 8, 0, "Salt values from the -wal"); + print_decode_line(aData,40+i, 8, 0, "Cksum over all prior fields"); +} + +/* +** Print the WalCkptInfo object +*/ +static void print_ckpt_info(unsigned char *aData){ + const int i = 96; + int j; + print_decode_line(aData, 0+i, 4, FG_NBO, "nBackfill"); + for(j=0; j<5; j++){ + char zLabel[100]; + sprintf(zLabel, "aReadMark[%d]", j); + print_decode_line(aData, 4*j+4+i, 4, FG_NBO, zLabel); + } + print_decode_line(aData,24+i, 8, 0, "aLock"); + print_decode_line(aData,32+i, 4, FG_NBO, "nBackfillAttempted"); + print_decode_line(aData,36+i, 4, FG_NBO, "notUsed0"); +} + + +int main(int argc, char **argv){ + unsigned char *aData; + if( argc<2 ){ + fprintf(stderr,"Usage: %s FILENAME\n", argv[0]); + exit(1); + } + fd = open(argv[1], O_RDONLY); + if( fd<0 ){ + fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); + exit(1); + } + aData = getContent(0, 136); + print_index_hdr(aData, 0); + print_index_hdr(aData, 1); + print_ckpt_info(aData); + free(aData); + close(fd); + return 0; +} From 9d9c41e25e460130678e5883fafaa13e97024add Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 03:40:15 +0000 Subject: [PATCH 065/156] Remove unnecessary NEVER() and ALWAYS() conditionals. FossilOrigin-Name: 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce --- manifest | 30 +++++++++++++++--------------- manifest.uuid | 2 +- src/build.c | 3 ++- src/insert.c | 3 ++- src/pcache.c | 19 +++++++++---------- src/prepare.c | 3 ++- src/select.c | 3 ++- src/vdbeaux.c | 2 +- src/vdbemem.c | 38 ++++++++++++++++++-------------------- src/walker.c | 19 +++++++++---------- src/where.c | 27 +++++++++++++-------------- src/wherecode.c | 5 +++-- 12 files changed, 77 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 362607f4d0..b25e49185c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sshowshm\sutility\sprogram\sfor\sprinting\sout\sthe\sshm\sheader\sin\sa\nhuman-readable\sformat. -D 2017-10-30T23:25:06.424 +C Remove\sunnecessary\sNEVER()\sand\sALWAYS()\sconditionals. +D 2017-10-31T03:40:15.885 F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d @@ -408,7 +408,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c e24b61144f9c9b15c4aa05954514190361061da721e56dcd1af6e0e945ee5909 +F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -425,7 +425,7 @@ F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40 +F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c @@ -453,17 +453,17 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa -F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c +F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 -F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a +F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 36345c63153e3e3e8ba533c6d3be6201c317358e8c2b34c51400abdf192515ab +F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -536,19 +536,19 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 -F src/vdbeaux.c c423065d50cee24bc8cba57764f5e9869a1bb920c50907f5dd363ebd7c5aef82 +F src/vdbeaux.c 9521a9364e68edad3c8d05ae63395d076724bed1c878c9b13fab61ada54e7d2a F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff +F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a -F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 -F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9 +F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f +F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 -F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1 +F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7 -R 5511878351634578cf57017f9253148a +P f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d +R 3a78a4e252d74365847a93117da52332 U drh -Z 5f57a6784d89256cd72e7dad10ab878c +Z 5327e6fab7ec504e303ebb0642487352 diff --git a/manifest.uuid b/manifest.uuid index a5b1cdf01a..0446940796 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d \ No newline at end of file +1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce \ No newline at end of file diff --git a/src/build.c b/src/build.c index a9428104a3..fc4dbcdba8 100644 --- a/src/build.c +++ b/src/build.c @@ -3852,9 +3852,10 @@ SrcList *sqlite3SrcListAppendFromTerm( goto append_from_error; } p = sqlite3SrcListAppend(db, p, pTable, pDatabase); - if( p==0 || NEVER(p->nSrc==0) ){ + if( p==0 ){ goto append_from_error; } + assert( p->nSrc>0 ); pItem = &p->a[p->nSrc-1]; assert( pAlias!=0 ); if( pAlias->n ){ diff --git a/src/insert.c b/src/insert.c index 5da52e7692..3a531495cf 100644 --- a/src/insert.c +++ b/src/insert.c @@ -909,7 +909,8 @@ void sqlite3Insert( VdbeOp *pOp; sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); pOp = sqlite3VdbeGetOp(v, -1); - if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ + assert( pOp!=0 ); + if( pOp->opcode==OP_Null && !IsVirtual(pTab) ){ appendFlag = 1; pOp->opcode = OP_NewRowid; pOp->p1 = iDataCur; diff --git a/src/pcache.c b/src/pcache.c index 6f43a1e8af..4b2e481c27 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -566,16 +566,15 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ */ void sqlite3PcacheMakeClean(PgHdr *p){ assert( sqlite3PcachePageSanity(p) ); - if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ - assert( (p->flags & PGHDR_CLEAN)==0 ); - pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); - p->flags |= PGHDR_CLEAN; - pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); - assert( sqlite3PcachePageSanity(p) ); - if( p->nRef==0 ){ - pcacheUnpin(p); - } + assert( (p->flags & PGHDR_DIRTY)!=0 ); + assert( (p->flags & PGHDR_CLEAN)==0 ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); + if( p->nRef==0 ){ + pcacheUnpin(p); } } diff --git a/src/prepare.c b/src/prepare.c index 8ff27fec6c..9141cb8a78 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -473,7 +473,8 @@ int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ */ assert( sqlite3_mutex_held(db->mutex) ); if( pSchema ){ - for(i=0; ALWAYS(inDb); i++){ + for(i=0; 1; i++){ + assert( inDb ); if( db->aDb[i].pSchema==pSchema ){ break; } diff --git a/src/select.c b/src/select.c index f31dd850ab..46c69d71fb 100644 --- a/src/select.c +++ b/src/select.c @@ -4342,7 +4342,8 @@ static int selectExpander(Walker *pWalker, Select *p){ if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ + assert( p->pSrc!=0 ); + if( (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } pTabList = p->pSrc; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6ec8b30680..e057a5df5f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3043,7 +3043,7 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ void sqlite3VdbeDelete(Vdbe *p){ sqlite3 *db; - if( NEVER(p==0) ) return; + assert( p!=0 ); db = p->db; assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); diff --git a/src/vdbemem.c b/src/vdbemem.c index 3c07f5a1a3..f9f58c43f4 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -350,26 +350,24 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ ** otherwise. */ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ - int rc = SQLITE_OK; - if( ALWAYS(pFunc && pFunc->xFinalize) ){ - sqlite3_context ctx; - Mem t; - assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - memset(&ctx, 0, sizeof(ctx)); - memset(&t, 0, sizeof(t)); - t.flags = MEM_Null; - t.db = pMem->db; - ctx.pOut = &t; - ctx.pMem = pMem; - ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( (pMem->flags & MEM_Dyn)==0 ); - if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); - memcpy(pMem, &t, sizeof(t)); - rc = ctx.isError; - } - return rc; + sqlite3_context ctx; + Mem t; + assert( pFunc!=0 ); + assert( pFunc->xFinalize!=0 ); + assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; + ctx.pMem = pMem; + ctx.pFunc = pFunc; + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ + assert( (pMem->flags & MEM_Dyn)==0 ); + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + memcpy(pMem, &t, sizeof(t)); + return ctx.isError; } /* diff --git a/src/walker.c b/src/walker.c index 49f9fa1897..ae7545bf5f 100644 --- a/src/walker.c +++ b/src/walker.c @@ -108,16 +108,15 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ struct SrcList_item *pItem; pSrc = p->pSrc; - if( ALWAYS(pSrc) ){ - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ - return WRC_Abort; - } - if( pItem->fg.isTabFunc - && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) - ){ - return WRC_Abort; - } + assert( pSrc!=0 ); + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + return WRC_Abort; + } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; } } return WRC_Continue; diff --git a/src/where.c b/src/where.c index 6021edf47f..4c1c332d2a 100644 --- a/src/where.c +++ b/src/where.c @@ -1863,22 +1863,21 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ ** Free a WhereInfo structure */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - WhereLevel *pLevel = &pWInfo->a[i]; - if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ - sqlite3DbFree(db, pLevel->u.in.aInLoop); - } + int i; + assert( pWInfo!=0 ); + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); } - sqlite3WhereClauseClear(&pWInfo->sWC); - while( pWInfo->pLoops ){ - WhereLoop *p = pWInfo->pLoops; - pWInfo->pLoops = p->pNextLoop; - whereLoopDelete(db, p); - } - sqlite3DbFreeNN(db, pWInfo); } + sqlite3WhereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + sqlite3DbFreeNN(db, pWInfo); } /* diff --git a/src/wherecode.c b/src/wherecode.c index 6cdf7b566b..da5c686a95 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -294,8 +294,8 @@ void sqlite3WhereAddScanStatus( */ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ int nLoop = 0; - while( ALWAYS(pTerm!=0) - && (pTerm->wtFlags & TERM_CODED)==0 + assert( pTerm!=0 ); + while( (pTerm->wtFlags & TERM_CODED)==0 && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) && (pLevel->notReady & pTerm->prereqAll)==0 ){ @@ -306,6 +306,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ } if( pTerm->iParent<0 ) break; pTerm = &pTerm->pWC->a[pTerm->iParent]; + assert( pTerm!=0 ); pTerm->nChild--; if( pTerm->nChild!=0 ) break; nLoop++; From 7eded5925b89222b4934a36a8c3d118dee4d7457 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 31 Oct 2017 12:01:01 +0000 Subject: [PATCH 066/156] Fix a couple of issues in incremental_index_check to do with CREATE INDEX statements that contain embedded SQL comments. FossilOrigin-Name: 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 --- ext/repair/checkindex.c | 146 ++++++++++++++++++++++++---------------- manifest | 14 ++-- manifest.uuid | 2 +- test/checkindex.test | 27 ++++++++ 4 files changed, 123 insertions(+), 66 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index fc91b74244..16fb3b327e 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -274,54 +274,6 @@ static void cidxFreeIndex(CidxIndex *pIdx){ } } -#define CIDX_PARSE_EOF 0 -#define CIDX_PARSE_COMMA 1 /* "," */ -#define CIDX_PARSE_OPEN 2 /* "(" */ -#define CIDX_PARSE_CLOSE 3 /* ")" */ - -static int cidxFindNext(const char *zIn, const char **pzOut){ - const char *z = zIn; - - while( 1 ){ - *pzOut = z; - switch( *z ){ - case '\0': - return CIDX_PARSE_EOF; - case '(': - return CIDX_PARSE_OPEN; - case ')': - return CIDX_PARSE_CLOSE; - case ',': - return CIDX_PARSE_COMMA; - - case '"': - case '\'': - case '`': { - char q = *z; - z++; - while( *z ){ - if( *z==q ){ - z++; - if( *z!=q ) break; - } - z++; - } - break; - } - - case '[': - while( *z++!=']' ); - break; - - default: - z++; - } - } - - assert( 0 ); - return -1; -} - static int cidx_isspace(char c){ return c==' ' || c=='\t' || c=='\r' || c=='\n'; } @@ -332,36 +284,114 @@ static int cidx_isident(char c){ || (c>='A' && c<='Z') || c=='_'; } +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +static int cidxFindNext( + const char *zIn, + const char **pzOut, + int *pbDoNotTrim /* OUT: True if prev is -- comment */ +){ + const char *z = zIn; + + while( 1 ){ + if( z[0]=='-' && z[1]=='-' ){ + z += 2; + while( z[0]!='\n' ){ + if( z[0]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + while( cidx_isspace(*z) ) z++; + *pbDoNotTrim = 1; + }else{ + *pzOut = z; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + case '/': + if( z[1]=='*' ){ + z += 2; + while( z[0]!='*' || z[1]!='/' ){ + if( z[1]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + z += 2; + break; + } + + default: + z++; + break; + } + *pbDoNotTrim = 0; + } + } + + assert( 0 ); + return -1; +} + static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ const char *z = zSql; const char *z1; int e; int rc = SQLITE_OK; int nParen = 1; + int bDoNotTrim = 0; CidxColumn *pCol = pIdx->aCol; - e = cidxFindNext(z, &z); + e = cidxFindNext(z, &z, &bDoNotTrim); if( e!=CIDX_PARSE_OPEN ) goto parse_error; z1 = z+1; z++; while( nParen>0 ){ - e = cidxFindNext(z, &z); + e = cidxFindNext(z, &z, &bDoNotTrim); if( e==CIDX_PARSE_EOF ) goto parse_error; if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ const char *z2 = z; if( pCol->zExpr ) goto parse_error; - while( cidx_isspace(z[-1]) ) z--; - if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ - z -= 3; - while( cidx_isspace(z[-1]) ) z--; - }else - if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ - z -= 4; + if( bDoNotTrim==0 ){ while( cidx_isspace(z[-1]) ) z--; + if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + while( cidx_isspace(z1[0]) ) z1++; } - while( cidx_isspace(z1[0]) ) z1++; pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); pCol++; z = z1 = z2+1; diff --git a/manifest b/manifest index 59ee792f38..974a630385 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check. -D 2017-10-30T19:38:41.046 +C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments. +D 2017-10-31T12:01:01.861 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185 +F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3 +F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 -R 32cef05dfb77ed0c73cb51913be9d3e3 +P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 +R 4a5b38f6ae42388faee270886368a889 U dan -Z 4d91b1f15edf1ab6cd873cde5a983874 +Z ee7435bc6b4be93625562670a465a6a7 diff --git a/manifest.uuid b/manifest.uuid index f363d67abb..be302e7d98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 \ No newline at end of file +2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index a91455558f..22463d3ffe 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -318,6 +318,33 @@ do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { {} {'{"x":5, "y":5}',5} } +#------------------------------------------------------------------------- +# +do_execsql_test 6.0 { + CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); + CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); + CREATE INDEX t6x2 ON t6(z, -- hello,world, + y); + + CREATE INDEX t6x3 ON t6(z -- hello,world + , y); + + INSERT INTO t6 VALUES(1, 2, 3); + INSERT INTO t6 VALUES(4, 5, 6); +} + +do_index_check_test 6.1 t6x1 { + {} 2,3,1 + {} 5,6,4 +} +do_index_check_test 6.2 t6x2 { + {} 3,2,1 + {} 6,5,4 +} +do_index_check_test 6.2 t6x3 { + {} 3,2,1 + {} 6,5,4 +} finish_test From 183f0aa6c4a72401ce4a00dc94f5f63744f10f6b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 12:06:29 +0000 Subject: [PATCH 067/156] Fix a minor comment typo. No changes to code. FossilOrigin-Name: 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b25e49185c..fe7d2f74fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\sNEVER()\sand\sALWAYS()\sconditionals. -D 2017-10-31T03:40:15.885 +C Fix\sa\sminor\scomment\stypo.\s\sNo\schanges\sto\scode. +D 2017-10-31T12:06:29.439 F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56 +F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d -R 3a78a4e252d74365847a93117da52332 +P 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce +R bec87929117a677c4c0a889fc21edace U drh -Z 5327e6fab7ec504e303ebb0642487352 +Z 4a41196ea5bc7a7d68e6bb73582e1035 diff --git a/manifest.uuid b/manifest.uuid index 0446940796..cec3b4159e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce \ No newline at end of file +5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 0785dd3b00..19c9ea0a08 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2154,7 +2154,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() ** to make a copy of the wal-index header into pWal->hdr. If the ** wal-index header has changed, *pChanged is set to 1 (as an indication -** to the caller that the local paget cache is obsolete and needs to be +** to the caller that the local page cache is obsolete and needs to be ** flushed.) When useWal==1, the wal-index header is assumed to already ** be loaded and the pChanged parameter is unused. ** From 6f099107791da2e83cd246e0ef248f4dfeb8f448 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 12:20:43 +0000 Subject: [PATCH 068/156] Fix redundancies in the makefiles that resulted from moving shell.c over to shell.c.in. FossilOrigin-Name: bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f --- Makefile.in | 2 +- Makefile.msc | 1 - main.mk | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index d03f009b79..7fef9f575b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -676,7 +676,7 @@ mptest: mptester$(TEXE) sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl - cp tsrc/shell.c tsrc/sqlite3ext.h . + cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . sqlite3ext.h: .target_source diff --git a/Makefile.msc b/Makefile.msc index fb6f61979e..d663b753bf 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1649,7 +1649,6 @@ mptest: mptester.exe sqlite3.c: .target_source sqlite3ext.h $(MKSQLITE3C_TOOL) $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) - copy tsrc\shell.c . copy $(TOP)\ext\session\sqlite3session.h . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl diff --git a/main.mk b/main.mk index 346dd8feb0..c74759b9cc 100644 --- a/main.mk +++ b/main.mk @@ -582,7 +582,7 @@ target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl - cp tsrc/shell.c tsrc/sqlite3ext.h . + cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c diff --git a/manifest b/manifest index fe7d2f74fe..f8ccb2f8d9 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sa\sminor\scomment\stypo.\s\sNo\schanges\sto\scode. -D 2017-10-31T12:06:29.439 -F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd +C Fix\sredundancies\sin\sthe\smakefiles\sthat\sresulted\sfrom\smoving\sshell.c\sover\nto\sshell.c.in. +D 2017-10-31T12:20:43.201 +F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d +F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -386,7 +386,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a6e3fa3c4d145189a67ad89e3f0d6541e8c085f5c34a21df0b1c870eb3aeb6ce +F main.mk 4fafbf2c23268ad5c497bd7c9da99dd3a53b1ffa2c150b6eb975397d4bb3f12f F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce -R bec87929117a677c4c0a889fc21edace +P 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e +R 2e8424d9fbc533001e55f43737738a2c U drh -Z 4a41196ea5bc7a7d68e6bb73582e1035 +Z 784be9d7f67c93467ca8ada4e99f0e81 diff --git a/manifest.uuid b/manifest.uuid index cec3b4159e..c8016ca992 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e \ No newline at end of file +bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f \ No newline at end of file From d59fde3da55e5b992030b5ed1a45ab921295458f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 14:56:44 +0000 Subject: [PATCH 069/156] Update the built procedures for the sqlite3_analyzer utility to allow it to be linked with an external sqlite3.o library. Automatically detect a missing dbstat extension and report the compile-time error. FossilOrigin-Name: bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/spaceanal.tcl | 17 +++++++++++++++-- tool/sqlite3_analyzer.c.in | 2 ++ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f8ccb2f8d9..649893539e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sredundancies\sin\sthe\smakefiles\sthat\sresulted\sfrom\smoving\sshell.c\sover\nto\sshell.c.in. -D 2017-10-31T12:20:43.201 +C Update\sthe\sbuilt\sprocedures\sfor\sthe\ssqlite3_analyzer\sutility\sto\sallow\sit\nto\sbe\slinked\swith\san\sexternal\ssqlite3.o\slibrary.\s\sAutomatically\sdetect\sa\nmissing\sdbstat\sextension\sand\sreport\sthe\scompile-time\serror. +D 2017-10-31T14:56:44.588 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1626,7 +1626,7 @@ F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a80 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl f40dc82b4d5e39d040a02a3ec38268e324068815e4292a15ffa30ee93208bbfd +F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e @@ -1635,7 +1635,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb -F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3 +F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e -R 2e8424d9fbc533001e55f43737738a2c +P bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f +R 405d6881e4ae5254162d311375149a09 U drh -Z 784be9d7f67c93467ca8ada4e99f0e81 +Z b66d68a4e4b6a1ef52955c34a93fec95 diff --git a/manifest.uuid b/manifest.uuid index c8016ca992..45e7b5b385 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f \ No newline at end of file +bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de \ No newline at end of file diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index 9db108dc26..3e08d3ffa7 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -1,7 +1,9 @@ -# Run this TCL script using "testfixture" in order get a report that shows -# how much disk space is used by a particular data to actually store data +# Run this TCL script using an SQLite-enabled TCL interpreter to get a report +# on how much disk space is used by a particular data to actually store data # versus how much space is unused. # +# The dbstat virtual table is required. +# if {[catch { @@ -147,6 +149,17 @@ if {$flags(-debug)} { db trace ::dbtrace } +# Make sure all required compile-time options are available +# +if {![db exists {SELECT 1 FROM pragma_compile_options + WHERE compile_options='ENABLE_DBSTAT_VTAB'}]} { + puts "The SQLite database engine linked with this application\ + lacks required capabilities. Recompile using the\ + -DSQLITE_ENABLE_DBSTAT_VTAB compile-time option to fix\ + this problem." + exit 1 +} + db eval {SELECT count(*) FROM sqlite_master} set pageSize [expr {wide([db one {PRAGMA page_size}])}] diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in index 547b22ffe1..e830521e08 100644 --- a/tool/sqlite3_analyzer.c.in +++ b/tool/sqlite3_analyzer.c.in @@ -14,7 +14,9 @@ #define SQLITE_DEFAULT_MEMSTATUS 0 #define SQLITE_MAX_EXPR_DEPTH 0 #define SQLITE_OMIT_LOAD_EXTENSION 1 +#ifndef USE_EXTERNAL_SQLITE INCLUDE sqlite3.c +#endif INCLUDE $ROOT/src/tclsqlite.c const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){ From 2737fa037660b4694ff5c190f9de955c15673d47 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 15:49:19 +0000 Subject: [PATCH 070/156] Add build infrastructure for the sqlite3_checker command-line utility. FossilOrigin-Name: 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 --- Makefile.in | 15 ++++++ Makefile.msc | 17 +++++++ ext/misc/btreeinfo.c | 4 +- ext/repair/sqlite3_checker.c.in | 31 ++++++++++++ ext/repair/sqlite3_checker.tcl | 88 +++++++++++++++++++++++++++++++++ main.mk | 16 ++++++ manifest | 22 +++++---- manifest.uuid | 2 +- 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 ext/repair/sqlite3_checker.c.in create mode 100644 ext/repair/sqlite3_checker.tcl diff --git a/Makefile.in b/Makefile.in index 5f6be57c37..74ade8241e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1187,6 +1187,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(TEXE): sqlite3_checker.c + $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) + dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 2804c25ab4..1e069cfe66 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1491,6 +1491,7 @@ TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ + sqlite3_checker.exe \ sqldiff.exe \ dbhash.exe @@ -2198,6 +2199,22 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@ + +sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 131c63e835..d75f06277f 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -61,7 +61,9 @@ ** SELECT name FROM sqlite_btreeinfo ** WHERE type='table' AND NOT hasRowid; */ -#include +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif SQLITE_EXTENSION_INIT1 #include #include diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in new file mode 100644 index 0000000000..af8b673ec5 --- /dev/null +++ b/ext/repair/sqlite3_checker.c.in @@ -0,0 +1,31 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH_INIT_PROC sqlite3_checker_init_proc +#define SQLITE_ENABLE_DBPAGE_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +INCLUDE sqlite3.c +INCLUDE $ROOT/src/tclsqlite.c +INCLUDE $ROOT/ext/misc/btreeinfo.c +INCLUDE $ROOT/ext/repair/checkindex.c + +#include + +const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ + sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); + return +BEGIN_STRING +INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl +END_STRING +; +} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl new file mode 100644 index 0000000000..86799741bc --- /dev/null +++ b/ext/repair/sqlite3_checker.tcl @@ -0,0 +1,88 @@ +# Read and run TCL commands from standard input. Used to implement +# the --tclsh option. +# This TCL script is the main driver script for the sqlite3_checker utility +# program. +# + +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 OPTIONS database-filename" + puts stderr { +Do sanity checking on a live SQLite3 database file specified by the +"database-filename" argument. + +Options: + + --tclsh Run the built-in TCL interpreter interactively (for debugging) + + --version Show the version number of SQLite +} + exit 1 +} + +set file_to_analyze {} +append argv {} +foreach arg $argv { + if {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } + if {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } + if {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } + if {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage + +# If a TCL script is specified on the command-line, then run that +# script. +# +if {[file extension $file_to_analyze]==".tcl"} { + source $file_to_analyze + exit 0 +} + +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} diff --git a/main.mk b/main.mk index c7711fb6e9..b9072a8dcc 100644 --- a/main.mk +++ b/main.mk @@ -464,6 +464,7 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ + sqlite3_checker$(EXE) \ sqldiff$(EXE) \ dbhash$(EXE) @@ -786,6 +787,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + tclsh $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(TEXE): sqlite3_checker.c + $(TCCX) $(TCL_FLAGS) sqlite3_checker.c -o $@ $(LIBTCL) $(THREADLIB) + dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index 974a630385..b28e60b264 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments. -D 2017-10-31T12:01:01.861 -F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 +C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility. +D 2017-10-31T15:49:19.426 +F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 +F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -258,7 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 +F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -329,6 +329,8 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 +F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92 +F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -387,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8 +F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1668,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 -R 4a5b38f6ae42388faee270886368a889 -U dan -Z ee7435bc6b4be93625562670a465a6a7 +P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 +R 0fe5d7d5728508c17e7a673dce852490 +U drh +Z 5b634bacf5fe6d374b5cc3f0a57d52b2 diff --git a/manifest.uuid b/manifest.uuid index be302e7d98..58aa8b845a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 \ No newline at end of file +429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 \ No newline at end of file From 2091f0fb3e52c493315796185b2329de6f55cfc6 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 15:55:48 +0000 Subject: [PATCH 071/156] Fix an error in the Makefile.msc for the sqlite3_checker.exe target. FossilOrigin-Name: 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 --- Makefile.msc | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 1e069cfe66..6d3f7c7ccd 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2209,7 +2209,7 @@ CHECKER_DEPS =\ $(TOP)/ext/repair/sqlite3_checker.c.in sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@ + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ diff --git a/manifest b/manifest index b28e60b264..20118f2662 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility. -D 2017-10-31T15:49:19.426 +C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget. +D 2017-10-31T15:55:48.444 F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0 +F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 -R 0fe5d7d5728508c17e7a673dce852490 +P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 +R d3e35133a6b7ed531196c48b12a56714 U drh -Z 5b634bacf5fe6d374b5cc3f0a57d52b2 +Z 22a9f7b25cabce39b5f911e9801c9b67 diff --git a/manifest.uuid b/manifest.uuid index 58aa8b845a..c6585cbdd7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 \ No newline at end of file +49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 \ No newline at end of file From 927dd51fce9e80feb2b3bda48bf9f77724cf0f89 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 18:09:40 +0000 Subject: [PATCH 072/156] Add the checkfreelist extension to the sqlite3_checker binary. FossilOrigin-Name: 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a --- Makefile.in | 1 + Makefile.msc | 1 + ext/repair/sqlite3_checker.c.in | 2 ++ main.mk | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 74ade8241e..0653ae8eae 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1193,6 +1193,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/Makefile.msc b/Makefile.msc index 6d3f7c7ccd..320dacc411 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2205,6 +2205,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index af8b673ec5..6045c36a9d 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -17,12 +17,14 @@ INCLUDE sqlite3.c INCLUDE $ROOT/src/tclsqlite.c INCLUDE $ROOT/ext/misc/btreeinfo.c INCLUDE $ROOT/ext/repair/checkindex.c +INCLUDE $ROOT/ext/repair/checkfreelist.c #include const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); return BEGIN_STRING INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl diff --git a/main.mk b/main.mk index b9072a8dcc..266328ac8b 100644 --- a/main.mk +++ b/main.mk @@ -793,6 +793,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/manifest b/manifest index 20118f2662..9b4d6af191 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget. -D 2017-10-31T15:55:48.444 -F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 +C Add\sthe\scheckfreelist\sextension\sto\sthe\ssqlite3_checker\sbinary. +D 2017-10-31T18:09:40.959 +F Makefile.in 750ff35a98dd7f4ad585923d88f4309bb4cebb51c4a6622a32f134a138c9bd13 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1 +F Makefile.msc 038f2c990966e6503f4960da574d4eef7b07c272a480983e10c4ebaa6d98cff9 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 -F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92 +F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -389,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030 +F main.mk 4f59e62614c9e2d793cdafa25f7b24f357b48b009099185c66c52af4e31ee36f F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 -R d3e35133a6b7ed531196c48b12a56714 +P 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 +R 0e924f8968e62617bb612a72ff226c2d U drh -Z 22a9f7b25cabce39b5f911e9801c9b67 +Z 18789d1f0a3682c9ed4749672ea58f7e diff --git a/manifest.uuid b/manifest.uuid index c6585cbdd7..99d2299f3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 \ No newline at end of file +0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a \ No newline at end of file From 4e8ad3bc1435e5202d1a59a9dd3e7eabfaff9469 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 00:10:34 +0000 Subject: [PATCH 073/156] Begin putting functionality into the sqlite3_checker binary. FossilOrigin-Name: e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 --- ext/repair/sqlite3_checker.tcl | 57 ++++++++++++++++++++++++++++++++++ manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index 86799741bc..fa3ffd73ce 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -4,6 +4,21 @@ # program. # +# Special case: +# +# sqlite3_checker --test FILENAME ARGS +# +# uses FILENAME in place of this script. +# +if {[lindex $argv 0]=="--test" && [llength $argv]>2} { + set file [lindex $argv 1] + set argv [lrange $argv 2 end] + source $file + exit 0 +} + +# Emulate a TCL shell +# proc tclsh {} { set line {} while {![eof stdin]} { @@ -27,6 +42,8 @@ proc tclsh {} { } } +# Print a usage message on standard error, then quit. +# proc usage {} { set argv0 [file rootname [file tail [info nameofexecutable]]] puts stderr "Usage: $argv0 OPTIONS database-filename" @@ -36,6 +53,8 @@ Do sanity checking on a live SQLite3 database file specified by the Options: + --freelist Perform a freelist check + --tclsh Run the built-in TCL interpreter interactively (for debugging) --version Show the version number of SQLite @@ -45,6 +64,8 @@ Options: set file_to_analyze {} append argv {} +set bFreelistCheck 0 +set bSummary 1 foreach arg $argv { if {[regexp {^-+tclsh$} $arg]} { tclsh @@ -56,6 +77,11 @@ foreach arg $argv { mem close exit 0 } + if {[regexp {^-+freelist$} $arg]} { + set bFreelistCheck 1 + set bSummary 0 + continue + } if {[regexp {^-} $arg]} { puts stderr "Unknown option: $arg" usage @@ -86,3 +112,34 @@ if {![file readable $root_filename]} { puts stderr "File is not readable: $root_filename" exit 1 } + +if {[catch {sqlite3 db $file_to_analyze} res]} { + puts stderr "Cannot open datababase $root_filename: $res" + exit 1 +} + +if {$bFreelistCheck} { + puts "freelist-check:" + flush stdout + puts [db one {SELECT checkfreelist('main')}] +} +if {$bSummary} { + set scale 0 + set pgsz [db one {PRAGMA page_size}] + db eval {SELECT nPage*$pgsz AS sz, name, tbl_name + FROM sqlite_btreeinfo + WHERE type='index' + ORDER BY 1 DESC, name} { + if {$scale==0} { + if {$sz>10000000} { + set scale 1000000.0 + set unit MB + } else { + set scale 1000.0 + set unit KB + } + } + puts [format {%7.1f %s index %s of table %s} \ + [expr {$sz/$scale}] $unit $name $tbl_name] + } +} diff --git a/manifest b/manifest index 0976eed231..e4e6ce9964 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2017-10-31T18:15:36.138 +C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary. +D 2017-11-01T00:10:34.144 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 +F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de -R 6d1ce2918085f767aa6a41a268922785 +P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 +R f819f4cdc6b92e91586a5e2d567aa2dc U drh -Z 59d272efca4b8d35d600b1005fb03cf7 +Z e7d1ee3ba646c59ecaada1f36dfcbcb9 diff --git a/manifest.uuid b/manifest.uuid index 462cf20d71..f800dc1ed5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 \ No newline at end of file +e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 \ No newline at end of file From 1fae37319e5a104161f9aa0d27e27508ac453ffa Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 01:01:20 +0000 Subject: [PATCH 074/156] Actually perform index checks when running sqlite3_checker FossilOrigin-Name: 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 --- ext/repair/sqlite3_checker.tcl | 113 ++++++++++++++++++++++++++++++--- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 112 insertions(+), 15 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index fa3ffd73ce..d4aa84aa3b 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -42,6 +42,37 @@ proc tclsh {} { } } +# Do an incremental integrity check of a single index +# +proc check_index {idxname batchsize} { + set i 0 + set more 1 + set nerr 0 + puts -nonewline "$idxname: " + while {$more} { + set more 0 + db eval {SELECT errmsg, current_key AS key + FROM incremental_index_check($idxname) + WHERE after_key=$key + LIMIT $batchsize} { + set more 1 + if {$errmsg!=""} { + if {$nerr>0} { + puts -nonewline "$idxname: " + } + incr nerr + puts "row $i: $errmsg" + } + incr i + } + } + if {$nerr==0} { + puts "$i entries, ok" + } else { + puts "$idxname: $nerr errors out of $i entries" + } +} + # Print a usage message on standard error, then quit. # proc usage {} { @@ -53,11 +84,19 @@ Do sanity checking on a live SQLite3 database file specified by the Options: - --freelist Perform a freelist check + --batchsize N Number of rows to check per transaction - --tclsh Run the built-in TCL interpreter interactively (for debugging) + --freelist Perform a freelist check - --version Show the version number of SQLite + --index NAME Run a check of the index NAME + + --summary Print summary information about the database + + --table NAME Run a check of all indexes for table NAME + + --tclsh Run the built-in TCL interpreter (for debugging) + + --version Show the version number of SQLite } exit 1 } @@ -65,8 +104,14 @@ Options: set file_to_analyze {} append argv {} set bFreelistCheck 0 -set bSummary 1 -foreach arg $argv { +set bSummary 0 +set zIndex {} +set zTable {} +set batchsize 100 +set bAll 1 +set argc [llength $argv] +for {set i 0} {$i<$argc} {incr i} { + set arg [lindex $argv $i] if {[regexp {^-+tclsh$} $arg]} { tclsh exit 0 @@ -79,7 +124,41 @@ foreach arg $argv { } if {[regexp {^-+freelist$} $arg]} { set bFreelistCheck 1 - set bSummary 0 + set bAll 0 + continue + } + if {[regexp {^-+summary$} $arg]} { + set bSummary 1 + set bAll 0 + continue + } + if {[regexp {^-+batchsize$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set batchsize [lindex $argv $i] + continue + } + if {[regexp {^-+index$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zIndex [lindex $argv $i] + set bAll 0 + continue + } + if {[regexp {^-+table$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zTable [lindex $argv $i] + set bAll 0 continue } if {[regexp {^-} $arg]} { @@ -118,8 +197,8 @@ if {[catch {sqlite3 db $file_to_analyze} res]} { exit 1 } -if {$bFreelistCheck} { - puts "freelist-check:" +if {$bFreelistCheck || $bAll} { + puts -nonewline "freelist-check: " flush stdout puts [db one {SELECT checkfreelist('main')}] } @@ -143,3 +222,21 @@ if {$bSummary} { [expr {$sz/$scale}] $unit $name $tbl_name] } } +if {$zIndex!=""} { + check_index $zIndex $batchsize +} +if {$zTable!=""} { + foreach idx [db eval {SELECT name FROM sqlite_master + WHERE type='index' AND rootpage>0 + AND tbl_name=$zTable}] { + check_index $idx $batchsize + } +} +if {$bAll} { + set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') + WHERE type='index' AND rootpage>0 + ORDER BY nEntry}] + foreach idx $allidx { + check_index $idx $batchsize + } +} diff --git a/manifest b/manifest index e4e6ce9964..a675da0f12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary. -D 2017-11-01T00:10:34.144 +C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker +D 2017-11-01T01:01:20.360 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967 +F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 -R f819f4cdc6b92e91586a5e2d567aa2dc +P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 +R 6f23f39130aeaf4ea9f27fe6626dd3eb U drh -Z e7d1ee3ba646c59ecaada1f36dfcbcb9 +Z cc97ad1013bf49d4724282a689bb05ce diff --git a/manifest.uuid b/manifest.uuid index f800dc1ed5..3e765dc206 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 \ No newline at end of file +54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 \ No newline at end of file From 24fa4d57a72de204c782e2e48664ae2484a7c2e8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 01:05:41 +0000 Subject: [PATCH 075/156] When sqlite3_checker finds a problem, show the row key as part of the error message, not the row index number. FossilOrigin-Name: 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df --- ext/repair/sqlite3_checker.tcl | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index d4aa84aa3b..25a5d252e6 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -61,7 +61,7 @@ proc check_index {idxname batchsize} { puts -nonewline "$idxname: " } incr nerr - puts "row $i: $errmsg" + puts "key($key): $errmsg" } incr i } diff --git a/manifest b/manifest index a675da0f12..222cd77113 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker -D 2017-11-01T01:01:20.360 +C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber. +D 2017-11-01T01:05:42.000 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e +F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 -R 6f23f39130aeaf4ea9f27fe6626dd3eb +P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 +R 84bd4abb8e94f0df6079dbff9fcf5a09 U drh -Z cc97ad1013bf49d4724282a689bb05ce +Z d157fe339c4385e31979ffec5d5c510b diff --git a/manifest.uuid b/manifest.uuid index 3e765dc206..69455d17b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 \ No newline at end of file +6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df \ No newline at end of file From 176b2a916bfa2d6577638713d4e3e3264132f122 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Nov 2017 06:59:19 +0000 Subject: [PATCH 076/156] Fix a race condition in os_unix.c that might allow a client to use a *-shm file corrupted by a power failure if another client fails between locking the *-shm file and truncating it to zero bytes. FossilOrigin-Name: d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 54 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ebf38789b5..91f071548a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. -D 2017-10-26T17:34:50.823 +C Fix\sa\srace\scondition\sin\sos_unix.c\sthat\smight\sallow\sa\sclient\sto\suse\sa\s*-shm\nfile\scorrupted\sby\sa\spower\sfailure\sif\sanother\sclient\sfails\sbetween\slocking\sthe\n*-shm\sfile\sand\struncating\sit\sto\szero\sbytes. +D 2017-11-01T06:59:19.745 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4 +F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 -R 49c1c99b22f8e84ff20a7afe5d1d547b +P f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 +R 37c523032e6b14675e1621a01cbce4c1 U dan -Z 07348874af35473af5ead94c9a86c9df +Z e56573f2b7ef574fffc6cf13c6dfb22b diff --git a/manifest.uuid b/manifest.uuid index 3da837219d..46c47f2d96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 \ No newline at end of file +d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7720a255ac..b94d417f49 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4372,6 +4372,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ + struct flock lock; int openFlags = O_RDWR | O_CREAT; if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ openFlags = O_RDONLY; @@ -4389,29 +4390,46 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Check to see if another process is holding the dead-man switch. - ** For a readonly_shm client, if no other process holds the DMS lock, - ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned. - ** Or, for a read-write connection, if no other process holds a - ** DMS lock the file is truncated to zero bytes in size. */ + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ rc = SQLITE_OK; - if( pShmNode->isReadonly ){ - struct flock lock; - lock.l_whence = SEEK_SET; - lock.l_start = UNIX_SHM_DMS; - lock.l_len = 1; - lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { - rc = SQLITE_IOERR_LOCK; - }else if( lock.l_type==F_UNLCK ){ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ rc = SQLITE_CANTOPEN_DIRTYWAL; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + } } - }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; } + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); } if( rc ) goto shm_open_err; From 02c21b7e34a35755b3e90bf5fb352af46e7df187 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 13:09:02 +0000 Subject: [PATCH 077/156] Minor cleanup in checkindex.c. Add progress displays when checking a single index in the top-level TCL script for sqlite3_checker. FossilOrigin-Name: 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 --- ext/repair/checkindex.c | 17 ++++++++++++----- ext/repair/sqlite3_checker.tcl | 25 ++++++++++++++----------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 16fb3b327e..362e199168 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -42,6 +42,7 @@ struct CidxTable { struct CidxCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_int64 iRowid; sqlite3_stmt *pStmt; }; @@ -93,6 +94,10 @@ static int cidxConnect( int rc = SQLITE_OK; CidxTable *pRet; +#define IIC_ERRMSG 0 +#define IIC_CURRENT_KEY 1 +#define IIC_INDEX_NAME 2 +#define IIC_AFTER_KEY 3 rc = sqlite3_declare_vtab(db, "CREATE TABLE xyz(" " errmsg TEXT, current_key TEXT," @@ -130,10 +135,10 @@ static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ if( p->usable==0 ) continue; if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - if( p->iColumn==2 ){ + if( p->iColumn==IIC_INDEX_NAME ){ iIdxName = i; } - if( p->iColumn==3 ){ + if( p->iColumn==IIC_AFTER_KEY ){ iAfterKey = i; } } @@ -193,6 +198,7 @@ static int cidxNext(sqlite3_vtab_cursor *pCursor){ cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); } }else{ + pCsr->iRowid++; rc = SQLITE_OK; } return rc; @@ -737,6 +743,7 @@ static int cidxFilter( assert( rc==SQLITE_OK ); rc = cidxNext(pCursor); } + pCsr->iRowid = 1; return rc; } @@ -749,8 +756,8 @@ static int cidxColumn( int iCol ){ CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol==0 || iCol==1 ); - if( iCol==0 ){ + assert( iCol>=IIC_ERRMSG && iCol<=IIC_AFTER_KEY ); + if( iCol==IIC_ERRMSG ){ const char *zVal = 0; if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ @@ -760,7 +767,7 @@ static int cidxColumn( zVal = "row missing"; } sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - }else{ + }else if( iCol==IIC_CURRENT_KEY ){ sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); } return SQLITE_OK; diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index 25a5d252e6..a29d0332a6 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -48,7 +48,11 @@ proc check_index {idxname batchsize} { set i 0 set more 1 set nerr 0 - puts -nonewline "$idxname: " + set pct 00.0 + set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') + WHERE name=$idxname}] + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout while {$more} { set more 0 db eval {SELECT errmsg, current_key AS key @@ -57,20 +61,19 @@ proc check_index {idxname batchsize} { LIMIT $batchsize} { set more 1 if {$errmsg!=""} { - if {$nerr>0} { - puts -nonewline "$idxname: " - } incr nerr - puts "key($key): $errmsg" + puts "$idxname: key($key): $errmsg" } incr i } + set x [format {%.1f} [expr {($i*100.0)/$max}]] + if {$x!=$pct} { + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout + set pct $x + } } - if {$nerr==0} { - puts "$i entries, ok" - } else { - puts "$idxname: $nerr errors out of $i entries" - } + puts "$idxname: $nerr errors out of $i entries" } # Print a usage message on standard error, then quit. @@ -107,7 +110,7 @@ set bFreelistCheck 0 set bSummary 0 set zIndex {} set zTable {} -set batchsize 100 +set batchsize 1000 set bAll 1 set argc [llength $argv] for {set i 0} {$i<$argc} {incr i} { diff --git a/manifest b/manifest index 222cd77113..bf8e3e2133 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber. -D 2017-11-01T01:05:42.000 +C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker. +D 2017-11-01T13:09:02.677 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -328,9 +328,9 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 +F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7 +F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 -R 84bd4abb8e94f0df6079dbff9fcf5a09 +P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df +R 9a8d975a41c4365cfebb7312db115daa U drh -Z d157fe339c4385e31979ffec5d5c510b +Z d02ff4965492ed83b018c8e578de7329 diff --git a/manifest.uuid b/manifest.uuid index 69455d17b6..86f32e4c10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df \ No newline at end of file +3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 \ No newline at end of file From 37ab9523fa97daea32e40e5ed111acf1f91dc78b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 18:05:32 +0000 Subject: [PATCH 078/156] Move the test scripts for checkfreelist and checkindex over into the ext/repair/test directory. Run them now using the sqlite3_checker utility with the --test option. Some tests are currently failing due to an incomplete port. This is an incremental check-in. FossilOrigin-Name: 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 --- ext/repair/sqlite3_checker.tcl | 8 +-- ext/repair/test/README.md | 13 ++++ .../repair/test/checkfreelist01.test | 33 +-------- .../repair/test/checkindex01.test | 38 +---------- ext/repair/test/test.tcl | 67 +++++++++++++++++++ manifest | 18 ++--- manifest.uuid | 2 +- 7 files changed, 98 insertions(+), 81 deletions(-) create mode 100644 ext/repair/test/README.md rename test/checkfreelist.test => ext/repair/test/checkfreelist01.test (75%) rename test/checkindex.test => ext/repair/test/checkindex01.test (90%) create mode 100644 ext/repair/test/test.tcl diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index a29d0332a6..ab8d05e600 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -1,5 +1,3 @@ -# Read and run TCL commands from standard input. Used to implement -# the --tclsh option. # This TCL script is the main driver script for the sqlite3_checker utility # program. # @@ -10,10 +8,10 @@ # # uses FILENAME in place of this script. # -if {[lindex $argv 0]=="--test" && [llength $argv]>2} { - set file [lindex $argv 1] +if {[lindex $argv 0]=="--test" && [llength $argv]>1} { + set ::argv0 [lindex $argv 1] set argv [lrange $argv 2 end] - source $file + source $argv0 exit 0 } diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md new file mode 100644 index 0000000000..8cc954adf5 --- /dev/null +++ b/ext/repair/test/README.md @@ -0,0 +1,13 @@ +To run these tests, first build sqlite3_checker: + + +> make sqlite3_checker + + +Then run the "test.tcl" script using: + + +> ./sqlite3_checker --test $path/test.tcl + + +Optionally add the full pathnames of individual *.test modules diff --git a/test/checkfreelist.test b/ext/repair/test/checkfreelist01.test similarity index 75% rename from test/checkfreelist.test rename to ext/repair/test/checkfreelist01.test index 93e4ecc234..7c33862583 100644 --- a/test/checkfreelist.test +++ b/ext/repair/test/checkfreelist01.test @@ -1,40 +1,12 @@ # 2017-10-11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the checkfreelist extension. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl set testprefix checkfreelist -ifcapable !vtab||!compound { - finish_test - return -} - -if {[file exists ../checkfreelist.so]==0} { - finish_test - return -} - do_execsql_test 1.0 { + PRAGMA page_size=1024; CREATE TABLE t1(a, b); } -db enable_load_extension 1 -do_execsql_test 1.1 { - SELECT load_extension('../checkfreelist.so'); -} {{}} - do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} do_execsql_test 1.3 { WITH s(i) AS ( @@ -118,6 +90,3 @@ do_execsql_test 1.11 { SELECT checkfreelist('main'); ROLLBACK; } {{leaf count out of range (249) on trunk page 5}} - -finish_test - diff --git a/test/checkindex.test b/ext/repair/test/checkindex01.test similarity index 90% rename from test/checkindex.test rename to ext/repair/test/checkindex01.test index 22463d3ffe..36bae886d2 100644 --- a/test/checkindex.test +++ b/ext/repair/test/checkindex01.test @@ -1,31 +1,7 @@ # 2017-10-11 # -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the checkindex extension. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl set testprefix checkindex -ifcapable !vtab||!compound { - finish_test - return -} - -if {[file exists ../checkindex.so]==0} { - finish_test - return -} - do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a); @@ -36,12 +12,7 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES('five', 10); CREATE INDEX i2 ON t1(a DESC); -} - -db enable_load_extension 1 -do_execsql_test 1.1 { - SELECT load_extension('../checkindex.so'); -} {{}} +} {} proc incr_index_check {idx nStep} { set Q { @@ -148,7 +119,7 @@ do_execsql_test 2.0 { CREATE INDEX i3 ON t2(b, c, d); CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); CREATE INDEX i5 ON t2(d, c DESC, b); -} +} {} do_index_check_test 2.1 i3 { {} NULL,1,1,1 @@ -205,7 +176,7 @@ do_execsql_test 3.0 { INSERT INTO t3 VALUES('a', 'b', NULL, 8); INSERT INTO t3 VALUES('a', 'b', NULL, 9); -} +} {} do_index_check_test 3.1 t3wxy { {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 @@ -345,6 +316,3 @@ do_index_check_test 6.2 t6x3 { {} 3,2,1 {} 6,5,4 } - -finish_test - diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl new file mode 100644 index 0000000000..c073bb73c5 --- /dev/null +++ b/ext/repair/test/test.tcl @@ -0,0 +1,67 @@ +# Run this script using +# +# sqlite3_checker --test $thisscript $testscripts +# +# The $testscripts argument is optional. If omitted, all *.test files +# in the same directory as $thisscript are run. +# +set NTEST 0 +set NERR 0 + + +# Invoke the do_test procedure to run a single test +# +# The $expected parameter is the expected result. The result is the return +# value from the last TCL command in $cmd. +# +# Normally, $expected must match exactly. But if $expected is of the form +# "/regexp/" then regular expression matching is used. If $expected is +# "~/regexp/" then the regular expression must NOT match. If $expected is +# of the form "#/value-list/" then each term in value-list must be numeric +# and must approximately match the corresponding numeric term in $result. +# Values must match within 10%. Or if the $expected term is A..B then the +# $result term must be in between A and B. +# +proc do_test {name cmd expected} { + if {[info exists ::testprefix]} { + set name "$::testprefix$name" + } + + incr ::NTEST + puts -nonewline $name... + flush stdout + + if {[catch {uplevel #0 "$cmd;\n"} result]} { + puts -nonewline $name... + puts "\nError: $result" + incr ::NERR + } else { + set ok [expr {[string compare $result $expected]==0}] + if {!$ok} { + puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" + incr ::NERR + } else { + puts " Ok" + } + } + flush stdout +} + +# +# do_execsql_test TESTNAME SQL RES +# +proc do_execsql_test {testname sql {result {}}} { + uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] +} + +if {[llength $argv]==0} { + set dir [file dirname $argv0] + set argv [glob -nocomplain $dir/*.test] +} +foreach testfile $argv { + file delete -force test.db + sqlite3 db test.db + source $testfile + catch {db close} +} +puts "$NERR errors out of $NTEST tests" diff --git a/manifest b/manifest index bf8e3e2133..ff5e3f094d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker. -D 2017-11-01T13:09:02.677 +C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in. +D 2017-11-01T18:05:32.697 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,11 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1 +F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 +F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e +F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f +F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7 +F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -655,8 +659,6 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a -F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1671,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df -R 9a8d975a41c4365cfebb7312db115daa +P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 +R 9a0909f5ab1dfc6f8f86ede9c43722ca U drh -Z d02ff4965492ed83b018c8e578de7329 +Z 17e4966dab1f438179fdac475786b16e diff --git a/manifest.uuid b/manifest.uuid index 86f32e4c10..1cd31e3773 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 \ No newline at end of file +17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 \ No newline at end of file From 3680e89b1e0fe927445e7046dee037957818fe1b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 18:31:34 +0000 Subject: [PATCH 079/156] Add the "sqlite3_imposter DB SCHEMA ROOT SQL" command to sqlite3_checker. Use it to fix the checkindex01.test module. There are still errors reported by the checkfreelist01.test module. FossilOrigin-Name: 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 --- ext/repair/sqlite3_checker.c.in | 52 +++++++++++++++++++++++++++++++ ext/repair/test/checkindex01.test | 29 +++++++---------- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index 6045c36a9d..96b15f2713 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -19,9 +19,61 @@ INCLUDE $ROOT/ext/misc/btreeinfo.c INCLUDE $ROOT/ext/repair/checkindex.c INCLUDE $ROOT/ext/repair/checkfreelist.c +/* +** Decode a pointer to an sqlite3 object. +*/ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ + struct SqliteDb *p; + Tcl_CmdInfo cmdInfo; + if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ + p = (struct SqliteDb*)cmdInfo.objClientData; + *ppDb = p->db; + return TCL_OK; + }else{ + *ppDb = 0; + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter +** sqlite3_imposter db main ;# rm all imposters +*/ +static int sqlite3_imposter( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + const char *zSchema; + int iRoot; + const char *zSql; + + if( objc!=3 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSchema = Tcl_GetString(objv[2]); + if( objc==3 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); + }else{ + if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; + zSql = Tcl_GetString(objv[4]); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); + } + return TCL_OK; +} + #include const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_imposter", + (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test index 36bae886d2..744a21c69e 100644 --- a/ext/repair/test/checkindex01.test +++ b/ext/repair/test/checkindex01.test @@ -71,16 +71,12 @@ do_index_check_test 1.4 i2 { do_test 1.5 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt1(a, b)} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 - - execsql { + sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} + db eval { UPDATE xt1 SET a='six' WHERE rowid=3; DELETE FROM xt1 WHERE rowid = 5; } - - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 1.6 i1 { @@ -211,16 +207,14 @@ do_index_check_test 4.1 t4cc { do_test 4.2 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - execsql { + db eval { UPDATE xt4 SET c1='hello' WHERE rowid=2; DELETE FROM xt4 WHERE rowid = 3; } - - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 4.3 t4cc { @@ -263,14 +257,13 @@ do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { do_test 5.2 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 - execsql { + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + db eval { UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; DELETE FROM xt5 WHERE rowid = 4; } - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 5.3.1 t5x { diff --git a/manifest b/manifest index ff5e3f094d..1e2056b04c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in. -D 2017-11-01T18:05:32.697 +C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule. +D 2017-11-01T18:31:34.119 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -329,11 +329,11 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 -F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 +F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f -F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7 +F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 -R 9a0909f5ab1dfc6f8f86ede9c43722ca +P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 +R 789d2ddc9f5813af408c7050793744d3 U drh -Z 17e4966dab1f438179fdac475786b16e +Z 00e34b3226a7a4c513950643ceed7a0f diff --git a/manifest.uuid b/manifest.uuid index 1cd31e3773..6e77603c23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 \ No newline at end of file +0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 \ No newline at end of file From 9b37b366c2624de4a15e216b34c3d4620bfa42da Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:22:08 +0000 Subject: [PATCH 080/156] Adjust page numbers in the ext/repair/test/checkfreelist01.test module due to the fact that the pending-byte page is no longer shifted down to a low-numbered page but is in its rightful place. FossilOrigin-Name: c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 --- ext/repair/test/checkfreelist01.test | 14 +++++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test index 7c33862583..7e2dd51c37 100644 --- a/ext/repair/test/checkfreelist01.test +++ b/ext/repair/test/checkfreelist01.test @@ -27,8 +27,8 @@ do_execsql_test 1.5 { ) SELECT i FROM freelist_trunk WHERE i!=1; } { - 10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235 - 4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5 + 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 + 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 } do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} @@ -50,7 +50,7 @@ do_execsql_test 1.7 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 1, get_int(data, 1)-1) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; } {{free-list count mismatch: actual=6725 header=6726}} @@ -59,19 +59,19 @@ do_execsql_test 1.8 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; -} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}} +} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} do_execsql_test 1.9 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 5, 0) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; -} {{leaf page 0 is out of range (child 3 of trunk page 4861)}} +} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} do_execsql_test 1.10 { BEGIN; diff --git a/manifest b/manifest index 1e2056b04c..4d5258d691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule. -D 2017-11-01T18:31:34.119 +C Adjust\spage\snumbers\sin\sthe\sext/repair/test/checkfreelist01.test\smodule\sdue\sto\nthe\sfact\sthat\sthe\spending-byte\spage\sis\sno\slonger\sshifted\sdown\sto\sa\slow-numbered\npage\sbut\sis\sin\sits\srightful\splace. +D 2017-11-01T19:22:08.991 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -332,7 +332,7 @@ F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89 F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e -F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f +F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 -R 789d2ddc9f5813af408c7050793744d3 +P 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 +R e97da7e6f34a0c98621feff75b5fc1d7 U drh -Z 00e34b3226a7a4c513950643ceed7a0f +Z 33cbf67c611e121ff7dcf902c08c95ea diff --git a/manifest.uuid b/manifest.uuid index 6e77603c23..6eea893785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 \ No newline at end of file +c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 \ No newline at end of file From 7dcde38f59eb0b5d858597147f7f8725d2672764 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:50:11 +0000 Subject: [PATCH 081/156] Make sure the JSON1 extension is available to sqlite3_checker.exe FossilOrigin-Name: a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de --- ext/repair/sqlite3_checker.c.in | 1 + manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index 96b15f2713..76e9708787 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -4,6 +4,7 @@ */ #define TCLSH_INIT_PROC sqlite3_checker_init_proc #define SQLITE_ENABLE_DBPAGE_VTAB 1 +#define SQLITE_ENABLE_JSON1 1 #undef SQLITE_THREADSAFE #define SQLITE_THREADSAFE 0 #undef SQLITE_ENABLE_COLUMN_METADATA diff --git a/manifest b/manifest index 86482e8b0c..47e05b0082 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\scheckindex.c\sextension\sand\sthe\ssqlite3_checker\sutility\sprogram\sused\nfor\sdoing\slive\svalidation\sof\slarge\sdatabases. -D 2017-11-01T19:44:19.216 +C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe +D 2017-11-01T19:50:11.569 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 -F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 +F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 @@ -1673,8 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 -R e97da7e6f34a0c98621feff75b5fc1d7 -T +closed c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 +P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 +R f83c5c1798f6f791a77e395a421e7535 U drh -Z 29d052b9d1f9214dbd3b25e5b7544e56 +Z b827e3b3bff13979721cde0c77e4ccc0 diff --git a/manifest.uuid b/manifest.uuid index b0b6dde090..05400f1a8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 \ No newline at end of file +a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de \ No newline at end of file From a48a290bcc6f7f81d0378aa5fc4ad51ee6e182cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:58:25 +0000 Subject: [PATCH 082/156] Fix harmless compiler warnings in sqlite3_checker.exe FossilOrigin-Name: 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 --- ext/repair/checkfreelist.c | 2 +- ext/repair/checkindex.c | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/dbpage.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c index cd2801e040..990be4afa7 100644 --- a/ext/repair/checkfreelist.c +++ b/ext/repair/checkfreelist.c @@ -166,7 +166,7 @@ static int checkFreelist( u32 i; u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - int nData = sqlite3_column_bytes(pTrunk, 1); + u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); u32 iNext = get4byte(&aData[0]); u32 nLeaf = get4byte(&aData[4]); diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 362e199168..fa8c713f7c 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -263,7 +263,7 @@ static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ char *cidxStrdup(int *pRc, const char *zStr){ char *zRet = 0; if( *pRc==SQLITE_OK ){ - int n = strlen(zStr); + int n = (int)strlen(zStr); zRet = cidxMalloc(pRc, n+1); if( zRet ) memcpy(zRet, zStr, n+1); } @@ -493,7 +493,7 @@ static int cidxDecodeAfter( ){ char **azAfter; int rc = SQLITE_OK; - int nAfterKey = strlen(zAfterKey); + int nAfterKey = (int)strlen(zAfterKey); azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 47e05b0082..9e0fd087b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe -D 2017-11-01T19:50:11.569 +C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe +D 2017-11-01T19:58:25.196 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -327,8 +327,8 @@ F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 -F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 +F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd +F ext/repair/checkindex.c 73f26fc1e2e17d68ede5db5b0aaf4869f2d6182f45f3d3624befc503c0f04a70 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e @@ -420,7 +420,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2 +F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 -R f83c5c1798f6f791a77e395a421e7535 +P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de +R 7ee45d8c7d2f00dd882d0691ca6a1bfb U drh -Z b827e3b3bff13979721cde0c77e4ccc0 +Z 0f28cc7179e20c43d298dd1935e3a35c diff --git a/manifest.uuid b/manifest.uuid index 05400f1a8f..7f5fcadcf3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de \ No newline at end of file +491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index f287e72229..75e4c42e67 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -329,7 +329,7 @@ static int dbpageUpdate( goto update_fail; } pBt = pTab->db->aDb[iDb].pBt; - if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){ + if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){ zErr = "bad page number"; goto update_fail; } From 92c02da33ec2717828eba13400355d43a7a6395f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Nov 2017 20:59:28 +0000 Subject: [PATCH 083/156] If a readonly_shm connection cannot map the *-shm file because no other process is holding the DMS lock, have it read from the database file only, ignoring any content in the wal file. FossilOrigin-Name: ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 --- manifest | 15 ++++--- manifest.uuid | 2 +- src/os_unix.c | 114 ++++++++++++++++++++++++++++------------------- src/wal.c | 24 +++++++--- test/walro2.test | 87 ++++++++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 59 deletions(-) create mode 100644 test/walro2.test diff --git a/manifest b/manifest index a6e4cc1fc1..45c7b16d67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2017-11-01T07:06:41.244 +C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile. +D 2017-11-01T20:59:28.295 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca +F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579 +F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,6 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a +F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1667,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de -R 46264ab14606a934f99609a277e3ff11 +P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 +R fcc128b567671c56015745eca2c28340 U dan -Z 7bfe9089058e9a41a8fafe2fd8ba1cb5 +Z e9ea9ea441f189aa52edada39fcf2f83 diff --git a/manifest.uuid b/manifest.uuid index 48096a84ae..ebf42c65a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 \ No newline at end of file +ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b94d417f49..e90e335cb7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4108,6 +4108,7 @@ struct unixShmNode { int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ @@ -4270,6 +4271,64 @@ static void unixShmPurge(unixFile *pFd){ } } +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTLOCK; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} + /* ** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. @@ -4308,7 +4367,7 @@ static void unixShmPurge(unixFile *pFd){ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShm *p = 0; /* The connection to be opened */ struct unixShmNode *pShmNode; /* The underlying mmapped file */ - int rc; /* Result code */ + int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ char *zShmFilename; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ @@ -4372,7 +4431,6 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - struct flock lock; int openFlags = O_RDWR | O_CREAT; if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ openFlags = O_RDONLY; @@ -4389,50 +4447,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** the original owner will not be able to connect. */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - - /* Use F_GETLK to determine the locks other processes are holding - ** on the DMS byte. If it indicates that another process is holding - ** a SHARED lock, then this process may also take a SHARED lock - ** and proceed with opening the *-shm file. - ** - ** Or, if no other process is holding any lock, then this process - ** is the first to open it. In this case take an EXCLUSIVE lock on the - ** DMS byte and truncate the *-shm file to zero bytes in size. Then - ** downgrade to a SHARED lock on the DMS byte. - ** - ** If another process is holding an EXCLUSIVE lock on the DMS byte, - ** return SQLITE_BUSY to the caller (it will try again). An earlier - ** version of this code attempted the SHARED lock at this point. But - ** this introduced a subtle race condition: if the process holding - ** EXCLUSIVE failed just before truncating the *-shm file, then this - ** process might open and use the *-shm file without truncating it. - ** And if the *-shm file has been corrupted by a power failure or - ** system crash, the database itself may also become corrupt. */ - rc = SQLITE_OK; - lock.l_whence = SEEK_SET; - lock.l_start = UNIX_SHM_DMS; - lock.l_len = 1; - lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { - rc = SQLITE_IOERR_LOCK; - }else if( lock.l_type==F_UNLCK ){ - if( pShmNode->isReadonly ){ - rc = SQLITE_CANTOPEN_DIRTYWAL; - }else{ - rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); - if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } - } - }else if( lock.l_type==F_WRLCK ){ - rc = SQLITE_BUSY; - } - if( rc==SQLITE_OK ){ - assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); - rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err; } } @@ -4456,7 +4473,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: @@ -4508,6 +4525,11 @@ static int unixShmMap( p = pDbFd->pShm; pShmNode = p->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); diff --git a/src/wal.c b/src/wal.c index 19c9ea0a08..1a11eb31e0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -575,9 +575,11 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - if( rc==SQLITE_READONLY ){ + if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; - rc = SQLITE_OK; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; + } } } } @@ -2084,6 +2086,14 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); if( rc!=SQLITE_OK ){ + if( rc==SQLITE_READONLY_CANTLOCK +#ifdef SQLITE_ENABLE_SNAPSHOT + && pWal->pSnapshot==0 +#endif + ){ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + rc = SQLITE_OK; + } return rc; }; assert( page0 || pWal->writeLock==0 ); @@ -2259,8 +2269,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } } - pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) ); + pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0; + if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame) #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) @@ -2272,7 +2284,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(0)); walShmBarrier(pWal); if( rc==SQLITE_OK ){ - if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + if( pInfo + && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, diff --git a/test/walro2.test b/test/walro2.test new file mode 100644 index 0000000000..2337776f27 --- /dev/null +++ b/test/walro2.test @@ -0,0 +1,87 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walro + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +do_multiclient_test tn { + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + do_test 1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); + } + file exists test.db-shm + } {1} + + do_test 1.2 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + + sql1 { SELECT * FROM t1 } + } {} + + do_test 1.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d} + + do_test 1.3.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} + +} + +finish_test From dea5ce36f57f83b4c6c78154c088d67ec8c5c594 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2017 11:12:03 +0000 Subject: [PATCH 084/156] Avoid locking shm-lock WAL_READ_LOCK(0) during recovery. Doing this allows recovery to proceed while a readonly_shm connection in unlocked mode has an ongoing read transaction. FossilOrigin-Name: 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 14 ++++++++++---- test/walro2.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 45c7b16d67..f3b7f8abe0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile. -D 2017-11-01T20:59:28.295 +C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction. +D 2017-11-02T11:12:03.045 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc +F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a -F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19 +F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 -R fcc128b567671c56015745eca2c28340 +P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 +R 5519605a9af9008f6aa3aade00e8ea39 U dan -Z e9ea9ea441f189aa52edada39fcf2f83 +Z aa82fa216e4468716131c3fe0543b69b diff --git a/manifest.uuid b/manifest.uuid index ebf42c65a3..08fbfd4ed2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 \ No newline at end of file +5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 1a11eb31e0..28fade96b0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1101,7 +1101,6 @@ static int walIndexRecover(Wal *pWal){ i64 nSize; /* Size of log file */ u32 aFrameCksum[2] = {0, 0}; int iLock; /* Lock offset to lock for checkpoint */ - int nLock; /* Number of locks to hold */ /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the @@ -1114,11 +1113,17 @@ static int walIndexRecover(Wal *pWal){ assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; - nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + if( rc==SQLITE_OK ){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc!=SQLITE_OK ){ + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + } + } if( rc ){ return rc; } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); @@ -1256,7 +1261,8 @@ finished: recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); - walUnlockExclusive(pWal, iLock, nLock); + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); return rc; } diff --git a/test/walro2.test b/test/walro2.test index 2337776f27..fb41d17f79 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -82,6 +82,37 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {a b c d} + code1 { db close } + code2 { db2 close } + code3 { db3 close } + + do_test 2.1 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES('e', 'f'); + INSERT INTO t1 VALUES('g', 'h'); + } + file exists test.db-shm + } {1} + + do_test 2.2 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + sql1 { + BEGIN; + SELECT * FROM t1; + } + } {a b c d} + + do_test 2.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d e f g h} + } finish_test From 44c8a97e019bfb791fd9b330212e2375d7516849 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2017 18:57:46 +0000 Subject: [PATCH 085/156] Fix test cases in wal2.test broken by the locking change in the previous commit. FossilOrigin-Name: f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal2.test | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f3b7f8abe0..5ad595579c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction. -D 2017-11-02T11:12:03.045 +C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit. +D 2017-11-02T18:57:46.036 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef +F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 -R 5519605a9af9008f6aa3aade00e8ea39 +P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d +R 30693e487c23aaa3c080ebb269e4e38f U dan -Z aa82fa216e4468716131c3fe0543b69b +Z a5242a9fca30f5b5641106b18cdb01ae diff --git a/manifest.uuid b/manifest.uuid index 08fbfd4ed2..a8bcfdb1fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d \ No newline at end of file +f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f \ No newline at end of file diff --git a/test/wal2.test b/test/wal2.test index bc170ad3ff..ddff71aed4 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -122,8 +122,8 @@ do_test wal2-1.1 { } {4 10} set RECOVER [list \ - {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ {4 1 lock shared} {4 1 unlock shared} \ @@ -393,8 +393,10 @@ tvfs delete set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer -lappend expected_locks {2 6 lock exclusive} ;# Lock recovery & all aReadMark[] -lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[] +lappend expected_locks {2 1 lock exclusive} ;# Lock recovery +lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] +lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery +lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer lappend expected_locks {3 1 lock exclusive} ;# Lock aReadMark[0] lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0] @@ -615,8 +617,8 @@ do_test wal2-6.4.1 { } {} set RECOVERY { - {0 1 lock exclusive} {1 7 lock exclusive} - {1 7 unlock exclusive} {0 1 unlock exclusive} + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} } set READMARK0_READ { {3 1 lock shared} {3 1 unlock shared} From 53be36b026469d665ed238dc4f65f37161862c7b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 3 Nov 2017 06:45:37 +0000 Subject: [PATCH 086/156] Fix harmless compiler warnings. FossilOrigin-Name: d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereexpr.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9e0fd087b8..c6be978982 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe -D 2017-11-01T19:58:25.196 +C Fix\sharmless\scompiler\swarnings. +D 2017-11-03T06:45:37.503 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -556,7 +556,7 @@ F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 -F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 +F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de -R 7ee45d8c7d2f00dd882d0691ca6a1bfb -U drh -Z 0f28cc7179e20c43d298dd1935e3a35c +P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 +R 2898c172e35d6c4a6ab6316ba23aa96d +U mistachkin +Z 0198e72b5c24cd8ac4cd7a2d9de73913 diff --git a/manifest.uuid b/manifest.uuid index 7f5fcadcf3..d96908a8ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 \ No newline at end of file +d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 9f83a84534..58f1908cf8 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -983,7 +983,7 @@ static void exprAnalyze( int op; /* Top-level operator. pExpr->op */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ - unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + unsigned char eOp2 = 0; /* op2 value for LIKE/REGEXP/GLOB */ int nLeft; /* Number of elements on left side vector */ if( db->mallocFailed ){ @@ -1227,7 +1227,7 @@ static void exprAnalyze( ** to do anything with MATCH functions. */ if( pWC->op==TK_AND ){ - Expr *pRight, *pLeft; + Expr *pRight = 0, *pLeft = 0; int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight); while( res-- > 0 ){ int idxNew; From 537e70289ff4168476252458965ca0bccf70559b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Nov 2017 08:46:48 +0000 Subject: [PATCH 087/156] The extensions functions in spellfix are all deterministic. FossilOrigin-Name: 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee --- ext/misc/spellfix.c | 24 +++++++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 1ac1712f4e..4f17b88e1d 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1122,15 +1122,17 @@ static int editDist3Install(sqlite3 *db){ if( pConfig==0 ) return SQLITE_NOMEM; memset(pConfig, 0, sizeof(*pConfig)); rc = sqlite3_create_function_v2(db, "editdist3", - 2, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); + 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, 0); if( rc==SQLITE_OK ){ rc = sqlite3_create_function_v2(db, "editdist3", - 3, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); + 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, 0); } if( rc==SQLITE_OK ){ rc = sqlite3_create_function_v2(db, "editdist3", - 1, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, - editDist3ConfigDelete); + 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, editDist3ConfigDelete); }else{ sqlite3_free(pConfig); } @@ -2895,18 +2897,22 @@ static sqlite3_module spellfix1Module = { static int spellfix1Register(sqlite3 *db){ int rc = SQLITE_OK; int i; - rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, - transliterateSqlFunc, 0, 0); + rc = sqlite3_create_function(db, "spellfix1_translit", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, + transliterateSqlFunc, 0, 0); if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_editdist", 2, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, editdistSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, phoneticHashSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, scriptCodeSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index c6be978982..a447c36649 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2017-11-03T06:45:37.503 +C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic. +D 2017-11-03T08:46:48.968 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -282,7 +282,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d +F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 -R 2898c172e35d6c4a6ab6316ba23aa96d -U mistachkin -Z 0198e72b5c24cd8ac4cd7a2d9de73913 +P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 +R b7838526febd368042ba32a4576c580c +U drh +Z 18ebbc1917110c35a4b2606f5ce6f533 diff --git a/manifest.uuid b/manifest.uuid index d96908a8ac..5035b0e5a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 \ No newline at end of file +29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee \ No newline at end of file From a3bc84255a1145a0f04359717fe7e35f70f88c41 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Nov 2017 08:53:37 +0000 Subject: [PATCH 088/156] Fix an LSM problem with read-only connections and compression (not compression-factory) callbacks. FossilOrigin-Name: 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e --- ext/lsm1/lsm_shared.c | 12 +++++------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c index c4bb9d6a23..2fdacf1eca 100644 --- a/ext/lsm1/lsm_shared.c +++ b/ext/lsm1/lsm_shared.c @@ -520,13 +520,11 @@ int lsmDbDatabaseConnect( ** recovery as necessary. Or, if this is a read-only database handle, ** defer attempting to connect to the system until a read-transaction ** is opened. */ - if( pDb->bReadonly==0 ){ - if( rc==LSM_OK ){ - rc = lsmFsConfigure(pDb); - } - if( rc==LSM_OK ){ - rc = doDbConnect(pDb); - } + if( rc==LSM_OK ){ + rc = lsmFsConfigure(pDb); + } + if( rc==LSM_OK && pDb->bReadonly==0 ){ + rc = doDbConnect(pDb); } return rc; diff --git a/manifest b/manifest index a447c36649..9a1f7ef8fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic. -D 2017-11-03T08:46:48.968 +C Fix\san\sLSM\sproblem\swith\sread-only\sconnections\sand\scompression\s(not\ncompression-factory)\scallbacks. +D 2017-11-04T08:53:37.599 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -245,7 +245,7 @@ F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709 F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea -F ext/lsm1/lsm_shared.c 1a76b7a5e89a003c24d58f1fb295c4203b48ef6acba9a194ac6003ade09fcd47 +F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525 F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 -R b7838526febd368042ba32a4576c580c -U drh -Z 18ebbc1917110c35a4b2606f5ce6f533 +P 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee +R 50deaa5f352753bc15393c76a6fab0d8 +U dan +Z 685560e0f9310b127a265df0e79cb145 diff --git a/manifest.uuid b/manifest.uuid index 5035b0e5a7..e247d77f4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee \ No newline at end of file +51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e \ No newline at end of file From 11caf4f4b73736f6b209d591d652c7e0398aa024 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Nov 2017 18:10:03 +0000 Subject: [PATCH 089/156] In cases where a readonly_shm client cannot take the DMS lock on the *-shm file, have it parse the wal file and create a wal-index to access it in heap memory. FossilOrigin-Name: 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 --- manifest | 16 ++--- manifest.uuid | 2 +- src/wal.c | 176 +++++++++++++++++++++++++++++++++++++++++++---- test/walro.test | 10 +-- test/walro2.test | 19 +++-- 5 files changed, 192 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 5ad595579c..7e5b9e5a86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit. -D 2017-11-02T18:57:46.036 +C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory. +D 2017-11-04T18:10:03.528 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae +F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1526,8 +1526,8 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a -F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776 +F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 +F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d -R 30693e487c23aaa3c080ebb269e4e38f +P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f +R 10fc0a1645ce290a2a283e11360fe887 U dan -Z a5242a9fca30f5b5641106b18cdb01ae +Z 79ea7f3203670e23f752126d91be88fd diff --git a/manifest.uuid b/manifest.uuid index a8bcfdb1fe..e8b4f82532 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f \ No newline at end of file +18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 28fade96b0..17e253a8b6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -455,6 +455,7 @@ struct Wal { u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + u8 bUnlocked; WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ @@ -1270,13 +1271,14 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - }else{ + } + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } } @@ -2091,15 +2093,13 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); + if( rc==SQLITE_READONLY_CANTLOCK ){ + assert( page0==0 && pWal->writeLock==0 ); + pWal->bUnlocked = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else if( rc!=SQLITE_OK ){ - if( rc==SQLITE_READONLY_CANTLOCK -#ifdef SQLITE_ENABLE_SNAPSHOT - && pWal->pSnapshot==0 -#endif - ){ - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - rc = SQLITE_OK; - } return rc; }; assert( page0 || pWal->writeLock==0 ); @@ -2116,7 +2116,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; @@ -2146,6 +2146,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } + if( pWal->bUnlocked ){ + if( rc!=SQLITE_OK ){ + walIndexClose(pWal, 0); + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } return rc; } @@ -2156,6 +2162,144 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ #define WAL_RETRY (-1) +/* +** Open an "unlocked" transaction. An unlocked transaction is a read +** transaction used by a read-only client in cases where the *-shm +** file cannot be mapped and its contents cannot be trusted. It is +** assumed that the *-wal file has been read and that a wal-index +** constructed in heap memory is currently available in Wal.apWiData[]. +** +** If this function returns SQLITE_OK, then the read transaction has +** been successfully opened. In this case output variable (*pChanged) +** is set to true before returning if the caller should discard the +** contents of the page cache before proceeding. Or, if it returns +** WAL_RETRY, then the heap memory wal-index has been discarded and +** the caller should retry opening the read transaction from the +** beginning (including attempting to map the *-shm file). +** +** If an error occurs, an SQLite error code is returned. +*/ +static int walBeginUnlocked(Wal *pWal, int *pChanged){ + i64 szWal; /* Size of wal file on disk in bytes */ + i64 iOffset; /* Current offset when reading wal file */ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + volatile void *pDummy; /* Dummy argument for xShmMap */ + int rc; /* Return code */ + u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ + + assert( pWal->bUnlocked ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Take WAL_READ_LOCK(0). This has the effect of preventing any + ** live clients from running a checkpoint, but does not stop them + ** from running recovery. */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + if( rc!=SQLITE_OK ){ + return (rc==SQLITE_BUSY ? WAL_RETRY : rc); + } + pWal->readLock = 0; + + /* Try to map the *-shm file again. If it succeeds this time, then + ** a non-readonly_shm connection has already connected to the database. + ** In this case, start over with opening the transaction. + ** + ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint + ** from taking place. But it does not prevent the wal from being wrapped + ** if a checkpoint has already taken place. This means that if another + ** client is connected at this point, it may have already checkpointed + ** the entire wal. In that case it would not be safe to continue with + ** the unlocked transaction, as the other client may overwrite wal + ** frames that this client is still using. */ + rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + if( rc!=SQLITE_READONLY_CANTLOCK ){ + assert( rc!=SQLITE_OK ); + rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); + goto begin_unlocked_out; + } + + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); + if( rc!=SQLITE_OK || (szWalhdr.mxFrame==0) ){ + /* If the wal file is too small to contain a wal-header and the + ** wal-index header has mxFrame==0, then it must be safe to proceed + ** reading the database file only. However, the page cache cannot + ** be trusted, as a read/write connection may have connected, written + ** the db, run a checkpoint, truncated the wal file and disconnected + ** since this client's last read transaction. */ + *pChanged = 1; + goto begin_unlocked_out; + } + + /* Check the salt keys at the start of the wal file still match. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto begin_unlocked_out; + } + if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + rc = WAL_RETRY; + goto begin_unlocked_out; + } + + /* Allocate a buffer to read frames into */ + szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( aFrame==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto begin_unlocked_out; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; + aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); + iOffset+szFrame<=szWal; + iOffset+=szFrame + ){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If this branch is taken, some other client has truncated the + ** *-wal file since the call to sqlite3OsFileSize() above. This + ** indicates that a read-write client has connected to the system. + ** So retry opening this read transaction. */ + rc = WAL_RETRY; + } + break; + } + if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + rc = WAL_RETRY; + break; + } + } + pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; + pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; + + begin_unlocked_out: + sqlite3_free(aFrame); + if( rc!=SQLITE_OK ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void*)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + pWal->bUnlocked = 0; + sqlite3WalEndReadTransaction(pWal); + *pChanged = 1; + } + return rc; +} + /* ** Attempt to start a read transaction. This might fail due to a race or ** other transient condition. When that happens, it returns WAL_RETRY to @@ -2244,7 +2388,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } if( !useWal ){ - rc = walIndexReadHdr(pWal, pChanged); + assert( rc==SQLITE_OK ); + if( pWal->bUnlocked==0 ){ + rc = walIndexReadHdr(pWal, pChanged); + } if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to @@ -2273,6 +2420,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( rc!=SQLITE_OK ){ return rc; } + else if( pWal->bUnlocked ){ + return walBeginUnlocked(pWal, pChanged); + } } assert( pWal->nWiData>0 ); @@ -2626,7 +2776,7 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || pWal->readLock==0 ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){ *piRead = 0; return SQLITE_OK; } diff --git a/test/walro.test b/test/walro.test index 09a33bbd3e..150344e151 100644 --- a/test/walro.test +++ b/test/walro.test @@ -105,7 +105,7 @@ do_multiclient_test tn { do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } list [catch { sql1 { SELECT * FROM t1 } } msg] $msg - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j}} do_test 1.2.3 { code1 { db close } @@ -114,10 +114,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_CANTOPEN} + } {SQLITE_OK} do_test 1.2.5 { file attributes test.db-shm -permissions rw-r--r-- @@ -162,10 +162,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j k l}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_CANTOPEN} + } {SQLITE_OK} #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent diff --git a/test/walro2.test b/test/walro2.test index fb41d17f79..ee4a341e24 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -62,7 +62,7 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 1.2 { + do_test 1.2.1 { forcecopy test.db test.db2 forcecopy test.db-wal test.db2-wal forcecopy test.db-shm test.db2-shm @@ -71,7 +71,10 @@ do_multiclient_test tn { } sql1 { SELECT * FROM t1 } - } {} + } {a b c d} + do_test 1.2.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} do_test 1.3.1 { code3 { sqlite3 db3 test.db2 } @@ -106,13 +109,21 @@ do_multiclient_test tn { BEGIN; SELECT * FROM t1; } - } {a b c d} + } {a b c d e f g h} do_test 2.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d e f g h} - + do_test 2.3.2 { + sql3 { INSERT INTO t1 VALUES('i', 'j') } + code3 { db3 close } + sql1 { COMMIT } + } {} + breakpoint + do_test 2.3.3 { + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} } finish_test From cbd3321978bd66bb53f87abe4e25c1f57b087652 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Nov 2017 21:06:35 +0000 Subject: [PATCH 090/156] Add further tests for the code added on this branch. FossilOrigin-Name: a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f --- manifest | 14 ++++----- manifest.uuid | 2 +- src/wal.c | 8 ++++- test/walro2.test | 76 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 89 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7e5b9e5a86..84f5558cad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory. -D 2017-11-04T18:10:03.528 +C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch. +D 2017-11-04T21:06:35.734 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb +F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 -F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54 +F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f -R 10fc0a1645ce290a2a283e11360fe887 +P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 +R 5345a5078e9be9411c860b392d7c8c77 U dan -Z 79ea7f3203670e23f752126d91be88fd +Z 9992407e85299b3993fc8148a1281ba1 diff --git a/manifest.uuid b/manifest.uuid index e8b4f82532..ce4b2816dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 \ No newline at end of file +a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 17e253a8b6..d8612ec850 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2253,6 +2253,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ } aData = &aFrame[WAL_FRAME_HDRSIZE]; + /* Check to see if a complete transaction has been appended to the + ** wal file since the heap-memory wal-index was created. If so, the + ** heap-memory wal-index is discarded and WAL_RETRY returned to + ** the caller. */ aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); @@ -2276,7 +2280,9 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ } if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; - /* If nTruncate is non-zero, this is a commit record. */ + /* If nTruncate is non-zero, then a complete transaction has been + ** appended to this wal file. Set rc to WAL_RETRY and break out of + ** the loop. */ if( nTruncate ){ rc = WAL_RETRY; break; diff --git a/test/walro2.test b/test/walro2.test index ee4a341e24..9a6b1a1b5c 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -15,7 +15,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl -set ::testprefix walro +source $testdir/wal_common.tcl +set ::testprefix walro2 # These tests are only going to work on unix. # @@ -120,10 +121,81 @@ do_multiclient_test tn { code3 { db3 close } sql1 { COMMIT } } {} - breakpoint do_test 2.3.3 { sql1 { SELECT * FROM t1 } } {a b c d e f g h i j} + + + #----------------------------------------------------------------------- + # 3.1.*: That a readonly_shm connection can read a database file if both + # the *-wal and *-shm files are zero bytes in size. + # + # 3.2.*: That it flushes the cache if, between transactions on a db with a + # zero byte *-wal file, some other connection modifies the db, then + # does "PRAGMA wal_checkpoint=truncate" to truncate the wal file + # back to zero bytes in size. + # + # 3.3.*: That, if between transactions some other process wraps the wal + # file, the readonly_shm client reruns recovery. + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + do_test 3.1.0 { + list [file exists test.db-wal] [file exists test.db-shm] + } {0 0} + do_test 3.1.1 { + close [open test.db-wal w] + close [open test.db-shm w] + code1 { + sqlite3 db file:test.db?readonly_shm=1 + } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h} + + do_test 3.2.0 { + list [file size test.db-wal] [file size test.db-shm] + } {0 0} + do_test 3.2.1 { + code2 { sqlite3 db2 test.db } + sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate } + code2 { db2 close } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2} + do_test 3.2.2 { + list [file size test.db-wal] [file size test.db-shm] + } {0 32768} + + do_test 3.3.0 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + } + code2 { db2 close } + code1 { db close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test 3.3.1 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10} + do_test 3.3.2 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA wal_checkpoint; + DELETE FROM t1; + INSERT INTO t1 VALUES('i', 'ii'); + } + code2 { db2 close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test 3.3.3 { + sql1 { SELECT * FROM t1 } + } {i ii} + } finish_test From 12f84e5ee55145c7427d071ed5afb2188e5c4ef3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Nov 2017 09:34:45 +0000 Subject: [PATCH 091/156] Fix a harmless compiler warning from Xcode 9.1. FossilOrigin-Name: 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/util.c | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9a1f7ef8fa..f1f80d7a12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sLSM\sproblem\swith\sread-only\sconnections\sand\scompression\s(not\ncompression-factory)\scallbacks. -D 2017-11-04T08:53:37.599 +C Fix\sa\sharmless\scompiler\swarning\sfrom\sXcode\s9.1. +D 2017-11-06T09:34:45.112 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -537,7 +537,7 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159 +F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee -R 50deaa5f352753bc15393c76a6fab0d8 -U dan -Z 685560e0f9310b127a265df0e79cb145 +P 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e +R ca8ae1ded89ca445023d13a31194e801 +U drh +Z c36ef70e8d55fddaef51df49c2bfea5f diff --git a/manifest.uuid b/manifest.uuid index e247d77f4b..0cd36e32cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e \ No newline at end of file +66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54 \ No newline at end of file diff --git a/src/util.c b/src/util.c index fbe3714b77..a4dbe8fdaf 100644 --- a/src/util.c +++ b/src/util.c @@ -387,12 +387,12 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ /* copy max significant digits to significand */ while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ @@ -405,7 +405,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ s = s*10 + (*z - '0'); d--; } - z+=incr, nDigits++; + z+=incr; nDigits++; } } if( z>=zEnd ) goto do_atof_calc; From ab548384525574b953fa7cec2a21b44b10d145e9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Nov 2017 19:49:34 +0000 Subject: [PATCH 092/156] Add further test cases for the new code on this branch. And a couple of fixes. FossilOrigin-Name: 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db --- manifest | 14 ++--- manifest.uuid | 2 +- src/wal.c | 20 +++---- test/walro2.test | 140 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 84f5558cad..1cc173595f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch. -D 2017-11-04T21:06:35.734 +C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes. +D 2017-11-06T19:49:34.916 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647 +F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 -F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f +F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 -R 5345a5078e9be9411c860b392d7c8c77 +P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f +R fb9f39269dd42d0221893523e00b9010 U dan -Z 9992407e85299b3993fc8148a1281ba1 +Z ae1d97ffe1b3072414f0d45a94e66fe3 diff --git a/manifest.uuid b/manifest.uuid index ce4b2816dd..d96dff9523 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f \ No newline at end of file +71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index d8612ec850..017e5e865c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2199,7 +2199,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** from running recovery. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); if( rc!=SQLITE_OK ){ - return (rc==SQLITE_BUSY ? WAL_RETRY : rc); + if( rc==SQLITE_BUSY ) rc = WAL_RETRY; + goto begin_unlocked_out; } pWal->readLock = 0; @@ -2223,7 +2224,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); - if( rc!=SQLITE_OK || (szWalhdr.mxFrame==0) ){ + if( rc!=SQLITE_OK ){ + goto begin_unlocked_out; + } + if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); goto begin_unlocked_out; } @@ -2268,16 +2273,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ /* Read and decode the next log frame. */ rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_SHORT_READ ){ - /* If this branch is taken, some other client has truncated the - ** *-wal file since the call to sqlite3OsFileSize() above. This - ** indicates that a read-write client has connected to the system. - ** So retry opening this read transaction. */ - rc = WAL_RETRY; - } - break; - } + if( rc!=SQLITE_OK ) break; if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; /* If nTruncate is non-zero, then a complete transaction has been diff --git a/test/walro2.test b/test/walro2.test index 9a6b1a1b5c..1c59b70ee6 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -196,6 +196,146 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {i ii} + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 4.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 4.1.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { SELECT * FROM t1 } + } {hello world} + + do_test 4.1.2 { + code3 { sqlite3 db3 test.db2 } + sql3 { + INSERT INTO t1 VALUES('!'); + PRAGMA wal_checkpoint = truncate; + } + code3 { db3 close } + } {} + do_test 4.1.3 { + sql2 { SELECT * FROM t1 } + } {hello world !} + + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 4.2.1 { + code1 { sqlite3 db test.db } + sql1 { + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('!'); + + PRAGMA cache_size = 10; + CREATE TABLE t2(x); + + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO t2 SELECT randomblob(500) FROM s; + SELECT count(*) FROM t2; + } + } {500} + do_test 4.2.2 { + file size test.db-wal + } {461152} + do_test 4.2.4 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + SELECT count(*) FROM t2; + } + } {hello world ! ! 0} + + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 5.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 5.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + } + } {hello world ! world hello} + + do_test 5.2 { + code1 { + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + set ::res2 [sql2 { SELECT * FROM t1 }] + } + puts "$msg xRead $args" + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + sqlite3 db file:test.db2?vfs=tvfs + db eval { SELECT * FROM sqlite_master } + + tvfs filter xRead + tvfs script handle_read + } + sql1 { + PRAGMA wal_checkpoint = truncate; + } + code1 { set ::res2 } + } {hello world ! world hello} + + do_test 5.3 { + code1 { db close } + code1 { tvfs delete } + } {} + + } finish_test From d24c5b1c9834e0bb0cbbc9f8731ef4a9332c1415 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 09:08:43 +0000 Subject: [PATCH 093/156] Add fault-injection tests for the code on this branch. FossilOrigin-Name: a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 --- manifest | 11 ++++---- manifest.uuid | 2 +- test/walrofault.test | 66 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 test/walrofault.test diff --git a/manifest b/manifest index 1cc173595f..dc34f6b3d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes. -D 2017-11-06T19:49:34.916 +C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch. +D 2017-11-07T09:08:43.757 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1528,6 +1528,7 @@ F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd +F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f -R fb9f39269dd42d0221893523e00b9010 +P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db +R 2894b2184d60284fd42b25cb64811e1a U dan -Z ae1d97ffe1b3072414f0d45a94e66fe3 +Z 3f08ac70c434ed263499224338c46e3a diff --git a/manifest.uuid b/manifest.uuid index d96dff9523..881ec0ad62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db \ No newline at end of file +a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 \ No newline at end of file diff --git a/test/walrofault.test b/test/walrofault.test new file mode 100644 index 0000000000..b396be218c --- /dev/null +++ b/test/walrofault.test @@ -0,0 +1,66 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix walro2 + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +db close +sqlite3_shutdown +sqlite3_config_uri 1 +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE t1(b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + PRAGMA cache_size = 10; + BEGIN; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) + INSERT INTO t1(b) SELECT randomblob(800) FROM s; +} {wal} +faultsim_save_and_close + +do_faultsim_test 1 -faults oom* -prep { + catch { db close } + faultsim_restore + sqlite3 db file:test.db?readonly_shm=1 +} -body { + execsql { SELECT * FROM t1 } +} -test { + faultsim_test_result {0 {hello world ! world hello}} +} + + + +finish_test From f12ba66cf8f6158965b22033c51703c0f6e1fde0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 15:43:52 +0000 Subject: [PATCH 094/156] On unix, if the *-shm file cannot be opened for read/write access, open it read-only and proceed as if the readonly_shm=1 URI option were specified. FossilOrigin-Name: ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 25 +++++++++++++------------ test/wal2.test | 2 +- test/walro.test | 6 +++++- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index dc34f6b3d8..d0bf050785 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch. -D 2017-11-07T09:08:43.757 +C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified. +D 2017-11-07T15:43:52.117 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7 +F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c +F test/wal2.test 2d81ffe2a02d9e5c7447b266f7153716cfcba7aecda5ed832db4544617399e29 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 +F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db -R 2894b2184d60284fd42b25cb64811e1a +P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 +R 7a4e57959f170e8ae96924256d8168d4 U dan -Z 3f08ac70c434ed263499224338c46e3a +Z fa20e28121f5e7127ed54b089c80ff82 diff --git a/manifest.uuid b/manifest.uuid index 881ec0ad62..6168e89f1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 \ No newline at end of file +ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e90e335cb7..c656215c4a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4369,7 +4369,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShmNode *pShmNode; /* The underlying mmapped file */ int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ + char *zShm; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ @@ -4410,14 +4410,14 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); - zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShmFilename, + sqlite3_snprintf(nShmFilename, zShm, SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); - sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; @@ -4431,15 +4431,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - int openFlags = O_RDWR | O_CREAT; - if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - openFlags = O_RDONLY; - pShmNode->isReadonly = 1; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777)); } - pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } /* If this process is running as root, make sure that the SHM file diff --git a/test/wal2.test b/test/wal2.test index ddff71aed4..a9cafec66d 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1130,7 +1130,7 @@ if {$::tcl_platform(platform) == "unix"} { foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} { 2 00644 00644 00644 1 1 1 3 00644 00400 00644 1 1 0 - 4 00644 00644 00400 1 0 0 + 4 00644 00644 00400 1 1 0 5 00400 00644 00644 1 1 0 7 00644 00000 00644 1 0 0 diff --git a/test/walro.test b/test/walro.test index 150344e151..cae52db6d3 100644 --- a/test/walro.test +++ b/test/walro.test @@ -139,11 +139,15 @@ do_multiclient_test tn { # Now check that if the readonly_shm option is not supplied, or if it # is set to zero, it is not possible to connect to the database without # read-write access to the shm. + # + # UPDATE: os_unix.c now opens the *-shm file in readonly mode + # automatically. + # do_test 1.3.1 { code1 { db close } code1 { sqlite3 db test.db } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j k l}} # Also test that if the -shm file can be opened for read/write access, # it is not if readonly_shm=1 is present in the URI. From 7031b12c570689fef665eb874667620f2af382ab Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Nov 2017 16:23:24 +0000 Subject: [PATCH 095/156] Enhance the checkindex.c virtual table so that it will output the index_name and after_key parameters. Also add a new diagnostic output column named scanner_sql which shows the SQL statement used to implement the current index scan. FossilOrigin-Name: 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87 --- ext/repair/checkindex.c | 238 +++++++++++++++++++----------- ext/repair/test/checkindex01.test | 24 ++- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 175 insertions(+), 103 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index fa8c713f7c..46ecc12fc4 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -14,16 +14,18 @@ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 +/* +** Stuff that is available inside the amalgamation, but which we need to +** declare ourselves if this module is compiled separately. +*/ #ifndef SQLITE_AMALGAMATION # include # include # include # include -# define ALWAYS(X) 1 -# define NEVER(X) 0 - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; #define get4byte(x) ( \ ((u32)((x)[0])<<24) + \ ((u32)((x)[1])<<16) + \ @@ -42,8 +44,10 @@ struct CidxTable { struct CidxCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ - sqlite3_int64 iRowid; - sqlite3_stmt *pStmt; + sqlite3_int64 iRowid; /* Row number of the output */ + char *zIdxName; /* Copy of the index_name parameter */ + char *zAfterKey; /* Copy of the after_key parameter */ + sqlite3_stmt *pStmt; /* SQL statement that generates the output */ }; typedef struct CidxColumn CidxColumn; @@ -82,7 +86,7 @@ static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ } /* -** Connect to then incremental_index_check virtual table. +** Connect to the incremental_index_check virtual table. */ static int cidxConnect( sqlite3 *db, @@ -98,10 +102,14 @@ static int cidxConnect( #define IIC_CURRENT_KEY 1 #define IIC_INDEX_NAME 2 #define IIC_AFTER_KEY 3 +#define IIC_SCANNER_SQL 4 rc = sqlite3_declare_vtab(db, "CREATE TABLE xyz(" - " errmsg TEXT, current_key TEXT," - " index_name HIDDEN, after_key HIDDEN" + " errmsg TEXT," /* Error message or NULL if everything is ok */ + " current_key TEXT," /* SQLite quote() text of key values */ + " index_name HIDDEN," /* IN: name of the index being scanned */ + " after_key HIDDEN," /* IN: Start scanning after this key */ + " scanner_sql HIDDEN" /* debuggingn info: SQL used for scanner */ ")" ); pRet = cidxMalloc(&rc, sizeof(CidxTable)); @@ -123,7 +131,15 @@ static int cidxDisconnect(sqlite3_vtab *pVtab){ } /* -** xBestIndex method. +** idxNum and idxStr are not used. There are only three possible plans, +** which are all distinguished by the number of parameters. +** +** No parameters: A degenerate plan. The result is zero rows. +** 1 Parameter: Scan all of the index starting with first entry +** 2 parameters: Scan the index starting after the "after_key". +** +** Provide successively smaller costs for each of these plans to encourage +** the query planner to select the one with the most parameters. */ static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ int iIdxName = -1; @@ -179,7 +195,8 @@ static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ static int cidxClose(sqlite3_vtab_cursor *pCursor){ CidxCursor *pCsr = (CidxCursor*)pCursor; sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; + sqlite3_free(pCsr->zIdxName); + sqlite3_free(pCsr->zAfterKey); sqlite3_free(pCsr); return SQLITE_OK; } @@ -650,6 +667,83 @@ static char *cidxColumnList( return zRet; } +/* +** Generate SQL (in memory obtained from sqlite3_malloc()) that will +** continue the index scan for zIdxName starting after zAfterKey. +*/ +int cidxGenerateScanSql( + CidxCursor *pCsr, /* The cursor which needs the new statement */ + const char *zIdxName, /* index to be scanned */ + const char *zAfterKey, /* start after this key, if not NULL */ + char **pzSqlOut /* OUT: Write the generated SQL here */ +){ + int rc; + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + char *zSrcList = 0; + char **azAfter = 0; + CidxIndex *pIdx = 0; + + *pzSqlOut = 0; + rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); + + zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); + zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); + zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); + zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); + zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); + + if( rc==SQLITE_OK && zAfterKey ){ + rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); + } + + if( rc || zAfterKey==0 ){ + *pzSqlOut = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zOrderBy + ); + }else{ + const char *zSep = ""; + char *zSql; + int i; + + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSubExpr, zTab, zSubWhere, zCurrentKey + ); + for(i=pIdx->nCol-1; i>=0; i--){ + int j; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; + for(j=0; j<2; j++){ + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zSrcList, zTab, zWhere, zOrderBy + ); + zSep = " UNION ALL "; + if( pIdx->aCol[i].bDesc==0 ) break; + } + } + *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); + } + + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + sqlite3_free(zSrcList); + cidxFreeIndex(pIdx); + sqlite3_free(azAfter); + return rc; +} + + /* ** Position a cursor back to the beginning. */ @@ -663,6 +757,13 @@ static int cidxFilter( const char *zIdxName = 0; const char *zAfterKey = 0; + sqlite3_free(pCsr->zIdxName); + pCsr->zIdxName = 0; + sqlite3_free(pCsr->zAfterKey); + pCsr->zAfterKey = 0; + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( argc>0 ){ zIdxName = (const char*)sqlite3_value_text(argv[0]); if( argc>1 ){ @@ -671,72 +772,13 @@ static int cidxFilter( } if( zIdxName ){ - char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - char *zSrcList = 0; - - char **azAfter = 0; - CidxIndex *pIdx = 0; - - rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); - - zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); - zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); - zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); - zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); - - if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); - } - - if( rc || zAfterKey==0 ){ - pCsr->pStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " - "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", - zSubExpr, zTab, zSubWhere, zCurrentKey, - zSrcList, zTab, zOrderBy - ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ - }else{ - const char *zSep = ""; - char *zSql; - int i; - - zSql = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", - zSubExpr, zTab, zSubWhere, zCurrentKey - ); - for(i=pIdx->nCol-1; i>=0; i--){ - int j; - if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; - for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, "%z" - "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", - zSql, zSep, zSrcList, zTab, zWhere, zOrderBy - ); - zSep = " UNION ALL "; - if( pIdx->aCol[i].bDesc==0 ) break; - } - } - zSql = cidxMprintf(&rc, "%z) AS i", zSql); - - /* printf("SQL: %s\n", zSql); */ + char *zSql = 0; + pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName); + pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0; + rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql); + if( zSql ){ pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); } - - sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(zSrcList); - cidxFreeIndex(pIdx); - sqlite3_free(azAfter); } if( pCsr->pStmt ){ @@ -756,26 +798,46 @@ static int cidxColumn( int iCol ){ CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol>=IIC_ERRMSG && iCol<=IIC_AFTER_KEY ); - if( iCol==IIC_ERRMSG ){ - const char *zVal = 0; - if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ - if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ - zVal = "row data mismatch"; + assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL ); + switch( iCol ){ + case IIC_ERRMSG: { + const char *zVal = 0; + if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ + if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ + zVal = "row data mismatch"; + } + }else{ + zVal = "row missing"; } - }else{ - zVal = "row missing"; + sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); + break; + } + case IIC_CURRENT_KEY: { + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); + break; + } + case IIC_INDEX_NAME: { + sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT); + break; + } + case IIC_AFTER_KEY: { + sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT); + break; + } + case IIC_SCANNER_SQL: { + char *zSql = 0; + cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql); + sqlite3_result_text(ctx, zSql, -1, sqlite3_free); + break; } - sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - }else if( iCol==IIC_CURRENT_KEY ){ - sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); } return SQLITE_OK; } /* Return the ROWID for the sqlite_btreeinfo table */ static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - *pRowid = 0; + CidxCursor *pCsr = (CidxCursor*)pCursor; + *pRowid = pCsr->iRowid; return SQLITE_OK; } diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test index 744a21c69e..d4abdcea7e 100644 --- a/ext/repair/test/checkindex01.test +++ b/ext/repair/test/checkindex01.test @@ -43,14 +43,24 @@ proc do_index_check_test {tn idx res} { } -do_execsql_test 1.2 { - SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); +do_execsql_test 1.2.1 { + SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1'); } { - 1 'five',5 - 1 'four',4 - 1 'one',1 - 1 'three',3 - 1 'two',2 + 1 1 'five',5 + 2 1 'four',4 + 3 1 'one',1 + 4 1 'three',3 + 5 1 'two',2 +} +do_execsql_test 1.2.2 { + SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql + FROM incremental_index_check('i1') LIMIT 1; +} { + 1 + 'five',5 + i1 + {} + {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' ORDER BY 1,2) AS i} } do_index_check_test 1.3 i1 { diff --git a/manifest b/manifest index f1f80d7a12..d1fed8a386 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning\sfrom\sXcode\s9.1. -D 2017-11-06T09:34:45.112 +C Enhance\sthe\scheckindex.c\svirtual\stable\sso\sthat\sit\swill\soutput\sthe\nindex_name\sand\safter_key\sparameters.\s\sAlso\sadd\sa\snew\sdiagnostic\soutput\ncolumn\snamed\sscanner_sql\swhich\sshows\sthe\sSQL\sstatement\sused\sto\simplement\nthe\scurrent\sindex\sscan. +D 2017-11-07T16:23:24.445 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -328,12 +328,12 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd -F ext/repair/checkindex.c 73f26fc1e2e17d68ede5db5b0aaf4869f2d6182f45f3d3624befc503c0f04a70 +F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 -F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14 +F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e -R ca8ae1ded89ca445023d13a31194e801 +P 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54 +R 9707d9dec21c954b720b6c37687c0a3c U drh -Z c36ef70e8d55fddaef51df49c2bfea5f +Z a2de00e6bfce7f49b15935d74b6b30ce diff --git a/manifest.uuid b/manifest.uuid index 0cd36e32cb..938db32b6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54 \ No newline at end of file +32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87 \ No newline at end of file From 0714a0dee19d2b72f6247fe59cf7c76f316eb4ca Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Nov 2017 16:54:20 +0000 Subject: [PATCH 096/156] Add the --trace option to the sqlite3_checker utility program. FossilOrigin-Name: dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d --- ext/repair/sqlite3_checker.tcl | 35 ++++++++++++++++++++++++++-------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index ab8d05e600..88c265f93c 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -42,7 +42,7 @@ proc tclsh {} { # Do an incremental integrity check of a single index # -proc check_index {idxname batchsize} { +proc check_index {idxname batchsize bTrace} { set i 0 set more 1 set nerr 0 @@ -51,18 +51,30 @@ proc check_index {idxname batchsize} { WHERE name=$idxname}] puts -nonewline "$idxname: $i of $max rows ($pct%)\r" flush stdout - while {$more} { - set more 0 - db eval {SELECT errmsg, current_key AS key + if {$bTrace} { + set sql {SELECT errmsg, current_key AS key, + CASE WHEN rowid=1 THEN scanner_sql END AS traceOut FROM incremental_index_check($idxname) WHERE after_key=$key - LIMIT $batchsize} { + LIMIT $batchsize} + } else { + set sql {SELECT errmsg, current_key AS key, NULL AS traceOut + FROM incremental_index_check($idxname) + WHERE after_key=$key + LIMIT $batchsize} + } + while {$more} { + set more 0 + db eval $sql { set more 1 if {$errmsg!=""} { incr nerr puts "$idxname: key($key): $errmsg" + } elseif {$traceOut!=""} { + puts "$idxname: $traceOut" } incr i + } set x [format {%.1f} [expr {($i*100.0)/$max}]] if {$x!=$pct} { @@ -97,6 +109,8 @@ Options: --tclsh Run the built-in TCL interpreter (for debugging) + --trace (Debugging only:) Output trace information on the scan + --version Show the version number of SQLite } exit 1 @@ -110,6 +124,7 @@ set zIndex {} set zTable {} set batchsize 1000 set bAll 1 +set bTrace 0 set argc [llength $argv] for {set i 0} {$i<$argc} {incr i} { set arg [lindex $argv $i] @@ -133,6 +148,10 @@ for {set i 0} {$i<$argc} {incr i} { set bAll 0 continue } + if {[regexp {^-+trace$} $arg]} { + set bTrace 1 + continue + } if {[regexp {^-+batchsize$} $arg]} { incr i if {$i>=$argc} { @@ -224,13 +243,13 @@ if {$bSummary} { } } if {$zIndex!=""} { - check_index $zIndex $batchsize + check_index $zIndex $batchsize $bTrace } if {$zTable!=""} { foreach idx [db eval {SELECT name FROM sqlite_master WHERE type='index' AND rootpage>0 AND tbl_name=$zTable}] { - check_index $idx $batchsize + check_index $idx $batchsize $bTrace } } if {$bAll} { @@ -238,6 +257,6 @@ if {$bAll} { WHERE type='index' AND rootpage>0 ORDER BY nEntry}] foreach idx $allidx { - check_index $idx $batchsize + check_index $idx $batchsize $bTrace } } diff --git a/manifest b/manifest index d1fed8a386..9879a2fe85 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\scheckindex.c\svirtual\stable\sso\sthat\sit\swill\soutput\sthe\nindex_name\sand\safter_key\sparameters.\s\sAlso\sadd\sa\snew\sdiagnostic\soutput\ncolumn\snamed\sscanner_sql\swhich\sshows\sthe\sSQL\sstatement\sused\sto\simplement\nthe\scurrent\sindex\sscan. -D 2017-11-07T16:23:24.445 +C Add\sthe\s--trace\soption\sto\sthe\ssqlite3_checker\sutility\sprogram. +D 2017-11-07T16:54:20.340 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f -F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 +F ext/repair/sqlite3_checker.tcl cc69e7fbc163f94da4a6400609be001543442d9f8f57a797d1eeb7b897585730 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54 -R 9707d9dec21c954b720b6c37687c0a3c +P 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87 +R d811e2909e0aaa74e9aeec168920f2dd U drh -Z a2de00e6bfce7f49b15935d74b6b30ce +Z c6a691c1b309b32f8cd47b4317d437a7 diff --git a/manifest.uuid b/manifest.uuid index 938db32b6b..c71454aad5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87 \ No newline at end of file +dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d \ No newline at end of file From 85e1f46eb93b58143dd8c3849b4d729694258fde Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 18:20:15 +0000 Subject: [PATCH 097/156] Fix a problem causing an INDEXED BY specifying an unusable partial index to be mishandled. FossilOrigin-Name: 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 12 +++++------- test/indexedby.test | 21 +++++++++++++++++++++ 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 9879a2fe85..e2118d2f81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--trace\soption\sto\sthe\ssqlite3_checker\sutility\sprogram. -D 2017-11-07T16:54:20.340 +C Fix\sa\sproblem\scausing\san\sINDEXED\sBY\sspecifying\san\sunusable\spartial\sindex\sto\sbe\nmishandled. +D 2017-11-07T18:20:15.108 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f -F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 +F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -959,7 +959,7 @@ F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7 F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721 -F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 +F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609 F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9 F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87 -R d811e2909e0aaa74e9aeec168920f2dd -U drh -Z c6a691c1b309b32f8cd47b4317d437a7 +P dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d +R da22ae68c1c6ec503942cfb34af1b16b +U dan +Z 75bfd7248538a99270ce42b4a5b4456f diff --git a/manifest.uuid b/manifest.uuid index c71454aad5..f26af75224 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d \ No newline at end of file +292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c1c332d2a..5545a45e87 100644 --- a/src/where.c +++ b/src/where.c @@ -2869,9 +2869,11 @@ static int whereLoopAddBtree( } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - /* Loop over all indices - */ - for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + /* Loop over all indices. If there was an INDEXED BY clause, then only + ** consider index pProbe. */ + for(; rc==SQLITE_OK && pProbe; + pProbe=(pSrc->pIBIndex ? 0 : pProbe->pNext), iSortIdx++ + ){ if( pProbe->pPartIdxWhere!=0 && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ @@ -2981,10 +2983,6 @@ static int whereLoopAddBtree( pBuilder->nRecValid = 0; pBuilder->pRec = 0; #endif - - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIBIndex ) break; } return rc; } diff --git a/test/indexedby.test b/test/indexedby.test index 83c7a5cccc..a0f7bea76a 100644 --- a/test/indexedby.test +++ b/test/indexedby.test @@ -364,4 +364,25 @@ do_eqp_test 11.10 { SELECT a,b,c FROM x2 INDEXED BY x2i WHERE a=1 AND b=1 AND c='3.0'; } {0 0 0 {SEARCH TABLE x2 USING COVERING INDEX x2i (a=? AND b=? AND rowid=?)}} +#------------------------------------------------------------------------- +# Check INDEXED BY works (throws an exception) with partial indexes that +# cannot be used. +do_execsql_test 12.1 { + CREATE TABLE o1(x INTEGER PRIMARY KEY, y, z); + CREATE INDEX p1 ON o1(z); + CREATE INDEX p2 ON o1(y) WHERE z=1; +} +do_catchsql_test 12.2 { + SELECT * FROM o1 INDEXED BY p2 ORDER BY 1; +} {1 {no query solution}} +do_execsql_test 12.3 { + DROP INDEX p1; + DROP INDEX p2; + CREATE INDEX p2 ON o1(y) WHERE z=1; + CREATE INDEX p1 ON o1(z); +} +do_catchsql_test 12.4 { + SELECT * FROM o1 INDEXED BY p2 ORDER BY 1; +} {1 {no query solution}} + finish_test From 0e90ad6a374f2747d54c8addad5a4a66b5401ed8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 19:02:00 +0000 Subject: [PATCH 098/156] Fix handling of partial indexes in checkindex.c (sqlite3_checker). FossilOrigin-Name: 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231 --- ext/repair/checkindex.c | 117 ++++++++++++++++++++---------- ext/repair/test/checkindex01.test | 32 +++++++- manifest | 14 ++-- manifest.uuid | 2 +- 4 files changed, 116 insertions(+), 49 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 46ecc12fc4..58706c1aab 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -59,6 +59,7 @@ struct CidxColumn { typedef struct CidxIndex CidxIndex; struct CidxIndex { + char *zWhere; /* WHERE clause, if any */ int nCol; /* Elements in aCol[] array */ CidxColumn aCol[1]; /* Array of indexed columns */ }; @@ -293,6 +294,7 @@ static void cidxFreeIndex(CidxIndex *pIdx){ for(i=0; inCol; i++){ sqlite3_free(pIdx->aCol[i].zExpr); } + sqlite3_free(pIdx->zWhere); sqlite3_free(pIdx); } } @@ -312,6 +314,24 @@ static int cidx_isident(char c){ #define CIDX_PARSE_OPEN 2 /* "(" */ #define CIDX_PARSE_CLOSE 3 /* ")" */ +/* +** Argument zIn points into the start, middle or end of a CREATE INDEX +** statement. If argument pbDoNotTrim is non-NULL, then this function +** scans the input until it finds EOF, a comma (",") or an open or +** close parenthesis character. It then sets (*pzOut) to point to said +** character and returns a CIDX_PARSE_XXX constant as appropriate. The +** parser is smart enough that special characters inside SQL strings +** or comments are not returned for. +** +** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut +** to point to the first character of the string that is not whitespace +** or part of an SQL comment and returns CIDX_PARSE_EOF. +** +** Additionally, if pbDoNotTrim is not NULL and the element immediately +** before (*pzOut) is an SQL comment of the form "-- comment", then +** (*pbDoNotTrim) is set before returning. In all other cases it is +** cleared. +*/ static int cidxFindNext( const char *zIn, const char **pzOut, @@ -320,6 +340,7 @@ static int cidxFindNext( const char *z = zIn; while( 1 ){ + while( cidx_isspace(*z) ) z++; if( z[0]=='-' && z[1]=='-' ){ z += 2; while( z[0]!='\n' ){ @@ -327,9 +348,18 @@ static int cidxFindNext( z++; } while( cidx_isspace(*z) ) z++; - *pbDoNotTrim = 1; + if( pbDoNotTrim ) *pbDoNotTrim = 1; + }else + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]!='*' || z[1]!='/' ){ + if( z[1]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + z += 2; }else{ *pzOut = z; + if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF; switch( *z ){ case '\0': return CIDX_PARSE_EOF; @@ -359,17 +389,6 @@ static int cidxFindNext( while( *z++!=']' ); break; - case '/': - if( z[1]=='*' ){ - z += 2; - while( z[0]!='*' || z[1]!='/' ){ - if( z[1]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - z += 2; - break; - } - default: z++; break; @@ -424,6 +443,14 @@ static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ z++; } + /* Search for a WHERE clause */ + cidxFindNext(z, &z, 0); + if( 0==sqlite3_strnicmp(z, "where", 5) ){ + pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]); + }else if( z[0]!='\0' ){ + goto parse_error; + } + return rc; parse_error: @@ -477,6 +504,7 @@ static int cidxLookupIndex( p->zExpr = 0; } pIdx->nCol = iCol; + pIdx->zWhere = 0; } cidxFinalize(&rc, pInfo); } @@ -700,36 +728,45 @@ int cidxGenerateScanSql( rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); } - if( rc || zAfterKey==0 ){ - *pzSqlOut = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " - "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", - zSubExpr, zTab, zSubWhere, zCurrentKey, - zSrcList, zTab, zOrderBy - ); - }else{ - const char *zSep = ""; - char *zSql; - int i; - - zSql = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", - zSubExpr, zTab, zSubWhere, zCurrentKey - ); - for(i=pIdx->nCol-1; i>=0; i--){ - int j; - if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; - for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, "%z" - "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", - zSql, zSep, zSrcList, zTab, zWhere, zOrderBy - ); - zSep = " UNION ALL "; - if( pIdx->aCol[i].bDesc==0 ) break; + if( rc==SQLITE_OK ){ + if( zAfterKey==0 ){ + *pzSqlOut = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zIdxName, + (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""), + zOrderBy + ); + }else{ + const char *zSep = ""; + char *zSql; + int i; + + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSubExpr, zTab, zSubWhere, zCurrentKey + ); + for(i=pIdx->nCol-1; i>=0; i--){ + int j; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; + for(j=0; j<2; j++){ + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (" + "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s" + ")", + zSql, zSep, zSrcList, zTab, zIdxName, + pIdx->zWhere ? pIdx->zWhere : "", + pIdx->zWhere ? " AND " : "", + zWhere, zOrderBy + ); + zSep = " UNION ALL "; + if( pIdx->aCol[i].bDesc==0 ) break; + } } + *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); } - *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); } sqlite3_free(zTab); diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test index d4abdcea7e..25930397d3 100644 --- a/ext/repair/test/checkindex01.test +++ b/ext/repair/test/checkindex01.test @@ -60,7 +60,7 @@ do_execsql_test 1.2.2 { 'five',5 i1 {} - {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' ORDER BY 1,2) AS i} + {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i} } do_index_check_test 1.3 i1 { @@ -319,3 +319,33 @@ do_index_check_test 6.2 t6x3 { {} 3,2,1 {} 6,5,4 } + +#------------------------------------------------------------------------- +# +do_execsql_test 7.0 { + CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z); + INSERT INTO t7 VALUES(1, 1, 1); + INSERT INTO t7 VALUES(2, 2, 0); + INSERT INTO t7 VALUES(3, 3, 1); + INSERT INTO t7 VALUES(4, 4, 0); + + CREATE INDEX t7i1 ON t7(y) WHERE z=1; + CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1; + CREATE INDEX t7i3 ON t7(y) WHERE -- yep + z=1; + CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep; +} +do_index_check_test 7.1 t7i1 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.2 t7i2 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.3 t7i3 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.4 t7i4 { + {} 1,1 {} 3,3 +} + + diff --git a/manifest b/manifest index e2118d2f81..004ac6c071 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\san\sINDEXED\sBY\sspecifying\san\sunusable\spartial\sindex\sto\sbe\nmishandled. -D 2017-11-07T18:20:15.108 +C Fix\shandling\sof\spartial\sindexes\sin\scheckindex.c\s(sqlite3_checker). +D 2017-11-07T19:02:00.173 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -328,12 +328,12 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd -F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67 +F ext/repair/checkindex.c 7d28c01a2e012ac64257d230fc452b2cafb78311a91a343633d01d95220f66f3 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl cc69e7fbc163f94da4a6400609be001543442d9f8f57a797d1eeb7b897585730 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 -F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4 +F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d -R da22ae68c1c6ec503942cfb34af1b16b +P 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468 +R 34ab99e7735ca564d873e9e5cddc35ee U dan -Z 75bfd7248538a99270ce42b4a5b4456f +Z e9b579d9d018f181ffb82f5d81fd8f28 diff --git a/manifest.uuid b/manifest.uuid index f26af75224..ab0b7bdaaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468 \ No newline at end of file +31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231 \ No newline at end of file From 08ecefc5b1bbcbcb3d04989daf454e8fa3007f82 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 21:15:07 +0000 Subject: [PATCH 099/156] Handle the race condition that may occur if another process connects and then checkpoints and truncates the wal file while a readonly-shm client is building its heap-memory wal-index. FossilOrigin-Name: 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/wal.c | 15 ++++++++------ test/walro2.test | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d0bf050785..bb8c19984a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified. -D 2017-11-07T15:43:52.117 +C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index. +D 2017-11-07T21:15:07.949 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20 +F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd +F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 -R 7a4e57959f170e8ae96924256d8168d4 +P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee +R e391f2931828dcd8c6e04450ea4ce5f1 U dan -Z fa20e28121f5e7127ed54b089c80ff82 +Z a1807096877d0d55e25dd1ebf350b50a diff --git a/manifest.uuid b/manifest.uuid index 6168e89f1d..011649cacf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee \ No newline at end of file +5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 017e5e865c..1a8bd74b02 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2071,6 +2071,12 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ return 0; } +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + /* ** Read the wal-index header from the wal-index and into pWal->hdr. ** If the wal-header appears to be corrupt, try to reconstruct the @@ -2149,6 +2155,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( pWal->bUnlocked ){ if( rc!=SQLITE_OK ){ walIndexClose(pWal, 0); + pWal->bUnlocked = 0; + assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } pWal->exclusiveMode = WAL_NORMAL_MODE; } @@ -2156,12 +2165,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ return rc; } -/* -** This is the value that walTryBeginRead returns when it needs to -** be retried. -*/ -#define WAL_RETRY (-1) - /* ** Open an "unlocked" transaction. An unlocked transaction is a read ** transaction used by a read-only client in cases where the *-shm diff --git a/test/walro2.test b/test/walro2.test index 1c59b70ee6..5a8ce023eb 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -335,7 +335,60 @@ do_multiclient_test tn { code1 { tvfs delete } } {} + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + do_test 6.1 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 6.2 { + code1 { + set ::nRem 5 + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + incr ::nRem -1 + if {$::nRem==0} { + code2 { sqlite3 db2 test.db2 } + sql2 { PRAGMA wal_checkpoint = truncate } + } + } + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + tvfs filter xRead + tvfs script handle_read + + sqlite3 db file:test.db2?readonly_shm=1&vfs=tvfs + db eval { SELECT * FROM t1 } + } + } {hello world ! world hello} + + do_test 6.3 { + code1 { db close } + code1 { tvfs delete } + } {} } finish_test From 6c9d8f640b5d2397e0f1338ae160f11220146d44 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 21:25:15 +0000 Subject: [PATCH 100/156] Update an assert in wal.c. FossilOrigin-Name: 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb8c19984a..0f9d93ff2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index. -D 2017-11-07T21:15:07.949 +C Update\san\sassert\sin\swal.c. +D 2017-11-07T21:25:15.188 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f +F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee -R e391f2931828dcd8c6e04450ea4ce5f1 +P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc +R 598ffeddb9515e7bdc3f854533f484ab U dan -Z a1807096877d0d55e25dd1ebf350b50a +Z 7844b4272e27592e14850347bc28da72 diff --git a/manifest.uuid b/manifest.uuid index 011649cacf..4a3a422204 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc \ No newline at end of file +94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 1a8bd74b02..4f8d5eadd9 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2844,8 +2844,8 @@ int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->minFrame>0 ); - for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ + assert( pWal->bUnlocked || pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; From 592eca1fda4a090df0b4e7107799fa07a45a519e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 02:50:09 +0000 Subject: [PATCH 101/156] Improved comments used for documentation of sqlite3_vfs. No changes to code. FossilOrigin-Name: db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 004ac6c071..16b3db4323 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\spartial\sindexes\sin\scheckindex.c\s(sqlite3_checker). -D 2017-11-07T19:02:00.173 +C Improved\scomments\sused\sfor\sdocumentation\sof\ssqlite3_vfs.\s\sNo\schanges\nto\scode. +D 2017-11-08T02:50:09.495 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -472,7 +472,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a -F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 +F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468 -R 34ab99e7735ca564d873e9e5cddc35ee -U dan -Z e9b579d9d018f181ffb82f5d81fd8f28 +P 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231 +R 3a2f607d35fb53b6e561d687398b02f3 +U drh +Z 38299a5cdd3519065be7d21e475b6bdf diff --git a/manifest.uuid b/manifest.uuid index ab0b7bdaaf..b7832db39a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231 \ No newline at end of file +db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9ff366304f..f2a379d55a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1131,12 +1131,18 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** in the name of the object stands for "virtual file system". See ** the [VFS | VFS documentation] for further information. ** -** The value of the iVersion field is initially 1 but may be larger in -** future versions of SQLite. Additional fields may be appended to this -** object when the iVersion value is increased. Note that the structure -** of the sqlite3_vfs object changes in the transaction between -** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not -** modified. +** The VFS interface is sometimes extended by adding new methods onto +** the end. Each time such an extension occurs, the iVersion field +** is incremented. The iVersion value started out as 1 in +** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2 +** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased +** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields +** may be appended to the sqlite3_vfs object and the iVersion value +** may increase again in future versions of SQLite. +** Note that the structure +** of the sqlite3_vfs object changes in the transition from +** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +** and yet the iVersion field was not modified. ** ** The szOsFile field is the size of the subclassed [sqlite3_file] ** structure used by this VFS. mxPathname is the maximum length of From 3e846cb81ff05c85c5a8e1a982a11f6b6d5d7764 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Nov 2017 11:14:53 +0000 Subject: [PATCH 102/156] Fix a problem causing LSM to add unnecessary padding to empty segments in compressed databases. FossilOrigin-Name: 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e --- ext/lsm1/Makefile | 4 ++-- ext/lsm1/lsm_file.c | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile index a4f8ebd367..7022b5682c 100644 --- a/ext/lsm1/Makefile +++ b/ext/lsm1/Makefile @@ -43,7 +43,7 @@ LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ # all: lsm.so -LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) +LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB lsm.so: $(LSMOBJ) $(TCCX) -shared -o lsm.so $(LSMOBJ) @@ -53,4 +53,4 @@ lsm.so: $(LSMOBJ) lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o # $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc - $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) + $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c index 3283c023dd..4001aac978 100644 --- a/ext/lsm1/lsm_file.c +++ b/ext/lsm1/lsm_file.c @@ -2803,7 +2803,7 @@ int lsmFsSortedPadding( Segment *pSeg ){ int rc = LSM_OK; - if( pFS->pCompress ){ + if( pFS->pCompress && pSeg->iFirst ){ Pgno iLast2; Pgno iLast = pSeg->iLastPg; /* Current last page of segment */ int nPad; /* Bytes of padding required */ diff --git a/manifest b/manifest index 16b3db4323..22a536158c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sused\sfor\sdocumentation\sof\ssqlite3_vfs.\s\sNo\schanges\nto\scode. -D 2017-11-08T02:50:09.495 +C Fix\sa\sproblem\scausing\sLSM\sto\sadd\sunnecessary\spadding\sto\sempty\ssegments\sin\ncompressed\sdatabases. +D 2017-11-08T11:14:53.660 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -211,7 +211,7 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c 635775226d07c743c770888a9dd5175afc6e67d3e28a4032b7fedc3bcaa92e65 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 -F ext/lsm1/Makefile 2951812df1c1cbc9e023af7e070876f479b3d75ce3898b3b9d00f83fecf13608 +F ext/lsm1/Makefile 98b0a24b45e248283d6bea4b6cb3e58d7b394edd8e96a0ac28c5fa5104813bad F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013 F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86 F ext/lsm1/lsm-test/lsmtest.h 5847594d4b43ec3412e1fd97104f7eb5fd770be55e691e6cb2e80929f86bebe3 @@ -240,7 +240,7 @@ F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f52 F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001 F ext/lsm1/lsmInt.h e9e5c5f08e35a104086102b3def94ee69cbc0d39002f6596f5c80a640439628e F ext/lsm1/lsm_ckpt.c ac6fb4581983291c2e0be6fbb68f12b26f0c08d606835c05417be1323d0fdd03 -F ext/lsm1/lsm_file.c e50f0e15427513dca4507110d0107268544bcaebb0ff0599a822da1f8d0da9eb +F ext/lsm1/lsm_file.c 4b3fb56336fbc9d941e1b2042e809d986feebdc41e73dc7fc4fdc0dd1bd4274d F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231 -R 3a2f607d35fb53b6e561d687398b02f3 -U drh -Z 38299a5cdd3519065be7d21e475b6bdf +P db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8 +R 6c8e575584f16f8416ee1b9774368bff +U dan +Z 515295e5c95123e6255cf7559983f082 diff --git a/manifest.uuid b/manifest.uuid index b7832db39a..bac99cd8b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8 \ No newline at end of file +1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e \ No newline at end of file From 7e45e3a5d76678356b88ceef773cdd6824008651 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 17:32:12 +0000 Subject: [PATCH 103/156] Change the name of SQLITE_READONLY_CANTLOCK to SQLITE_READONLY_CANTINIT. FossilOrigin-Name: 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 2 +- src/os_unix.c | 6 +++--- src/sqlite.h.in | 2 +- src/wal.c | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 0f9d93ff2b..7b0825f535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\san\sassert\sin\swal.c. -D 2017-11-07T21:25:15.188 +C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT. +D 2017-11-08T17:32:12.655 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -428,7 +428,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c +F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e +F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a -F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86 +F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2 +F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc -R 598ffeddb9515e7bdc3f854533f484ab -U dan -Z 7844b4272e27592e14850347bc28da72 +P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee +R a029a6995bf9493f5c27e7086cc8ad9a +U drh +Z d606e5b93c29b98849932178fdbf9f8a diff --git a/manifest.uuid b/manifest.uuid index 4a3a422204..3e854635d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee \ No newline at end of file +6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f \ No newline at end of file diff --git a/src/main.c b/src/main.c index 49613f6c74..d090845ae3 100644 --- a/src/main.c +++ b/src/main.c @@ -1314,7 +1314,7 @@ const char *sqlite3ErrName(int rc){ case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; diff --git a/src/os_unix.c b/src/os_unix.c index c656215c4a..c5d9aca2c6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4278,7 +4278,7 @@ static void unixShmPurge(unixFile *pFd){ ** ** If the DMS cannot be locked because this is a readonly_shm=1 ** connection and no other process already holds a lock, return -** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1. +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ struct flock lock; @@ -4311,7 +4311,7 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ }else if( lock.l_type==F_UNLCK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; - rc = SQLITE_READONLY_CANTLOCK; + rc = SQLITE_READONLY_CANTINIT; }else{ rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ @@ -4450,7 +4450,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); rc = unixLockSharedMemory(pDbFd, pShmNode); - if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err; + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0c0ed25b30..1d083cf06f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -511,7 +511,7 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) -#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) diff --git a/src/wal.c b/src/wal.c index 4f8d5eadd9..11cdeb8440 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2099,7 +2099,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); - if( rc==SQLITE_READONLY_CANTLOCK ){ + if( rc==SQLITE_READONLY_CANTINIT ){ assert( page0==0 && pWal->writeLock==0 ); pWal->bUnlocked = 1; pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; @@ -2219,7 +2219,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** the unlocked transaction, as the other client may overwrite wal ** frames that this client is still using. */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); - if( rc!=SQLITE_READONLY_CANTLOCK ){ + if( rc!=SQLITE_READONLY_CANTINIT ){ assert( rc!=SQLITE_OK ); rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); goto begin_unlocked_out; @@ -2509,7 +2509,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } if( mxI==0 ){ assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); From 7316960cec0ea2b411946964f4c02538ff7e3e1c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 17:51:10 +0000 Subject: [PATCH 104/156] Turns out that SQLITE_READONLY_CANTLOCK is an historical name that must be preserved. So make a new SQLITE_READLOCK_CANTINIT name instead. FossilOrigin-Name: 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7b0825f535..3f6afa398d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT. -D 2017-11-08T17:32:12.655 +C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead. +D 2017-11-08T17:51:10.682 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a -F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24 +F src/sqlite.h.in df0f3e18bd7210a562dfa4dc7c5b8f57058dfd0f51d771feb5fcfbe3d6ccae5b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee -R a029a6995bf9493f5c27e7086cc8ad9a +P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f +R 1af2a9a9afceb6f159be2ed742a0108b U drh -Z d606e5b93c29b98849932178fdbf9f8a +Z 83682bfec24bdede404f3ec79bae717f diff --git a/manifest.uuid b/manifest.uuid index 3e854635d3..70823ee687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f \ No newline at end of file +04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1d083cf06f..db6e0cbae3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -511,9 +511,10 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) -#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) #define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) #define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) From 9214c1efe831ca60df04f0a06bb6b1cfd5ca6d38 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 19:26:27 +0000 Subject: [PATCH 105/156] Extra comments on the sqlite3OsShmMap() call in walBeginUnlocked(). No changes to code. FossilOrigin-Name: 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f6afa398d..a22cee602e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead. -D 2017-11-08T17:51:10.682 +C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode. +D 2017-11-08T19:26:27.278 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e +F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f -R 1af2a9a9afceb6f159be2ed742a0108b +P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 +R eba2a72214a6da7d512bda6426cfe5de U drh -Z 83682bfec24bdede404f3ec79bae717f +Z 493ea33d970c37f0e10aaccf35cd6d47 diff --git a/manifest.uuid b/manifest.uuid index 70823ee687..a866afde61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 \ No newline at end of file +033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 11cdeb8440..afd0e3f7b6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2210,6 +2210,14 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ /* Try to map the *-shm file again. If it succeeds this time, then ** a non-readonly_shm connection has already connected to the database. ** In this case, start over with opening the transaction. + ** + ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never + ** return SQLITE_OK here, as that would imply that it had established + ** a read/write mapping. A return of SQLITE_READONLY means success - that + ** a mapping has been established to a shared-memory segment that is actively + ** maintained by a writer. SQLITE_READONLY_CANTINIT means that all + ** all connections to the -shm file are read-only and hence the content + ** of the -shm file might be out-of-date. ** ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint ** from taking place. But it does not prevent the wal from being wrapped @@ -2219,8 +2227,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** the unlocked transaction, as the other client may overwrite wal ** frames that this client is still using. */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ - assert( rc!=SQLITE_OK ); rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); goto begin_unlocked_out; } From 2fba394c97f9d9cf6b6c447dd5c2d01651593c73 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Nov 2017 03:55:09 +0000 Subject: [PATCH 106/156] Disallow ORDER BY and LIMIT on UPDATE and DELETE of views and WITHOUT ROWID tables. This is a temporary fix for ticket [d4beea1633f1b88f] until a better solution can be found. FossilOrigin-Name: 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 6 +++++- test/wherelimit.test | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 22a536158c..9a2dbc0bc9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\sLSM\sto\sadd\sunnecessary\spadding\sto\sempty\ssegments\sin\ncompressed\sdatabases. -D 2017-11-08T11:14:53.660 +C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound. +D 2017-11-09T03:55:09.531 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -468,7 +468,7 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 +F src/resolve.c 704978e5aabb9f524789e3b53016b4068a4e64c669c0f8ff025d2b23b95d62b5 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a @@ -1557,7 +1557,7 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 -F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 +F test/wherelimit.test baa8e5e30b2eddc395cbb3edb313deea14ca6938ea8a96c02a03c34d0ab84b5b F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8 -R 6c8e575584f16f8416ee1b9774368bff -U dan -Z 515295e5c95123e6255cf7559983f082 +P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e +R a028f3be0864f739aeb3787daf46fc8e +U drh +Z 97937a65b1ac3d4b961579eac7d25cbd diff --git a/manifest.uuid b/manifest.uuid index bac99cd8b8..e270b8e48e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e \ No newline at end of file +62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 4a1e8284c8..945654ead0 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -596,7 +596,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ SrcList *pSrcList = pNC->pSrcList; struct SrcList_item *pItem; assert( pSrcList && pSrcList->nSrc==1 ); - pItem = pSrcList->a; + pItem = pSrcList->a; + if( !HasRowid(pItem->pTab) || pItem->pTab->pSelect!=0 ){ + sqlite3ErrorMsg(pParse, "ORDER BY and LIMIT not support for table %s", + pItem->pTab->zName); + } pExpr->op = TK_COLUMN; pExpr->pTab = pItem->pTab; pExpr->iTable = pItem->iCursor; diff --git a/test/wherelimit.test b/test/wherelimit.test index f0cfbb61e8..e10d90b40b 100644 --- a/test/wherelimit.test +++ b/test/wherelimit.test @@ -279,6 +279,43 @@ ifcapable {update_delete_limit} { execsql {SELECT count(*) FROM t1 WHERE y=1} } {6} + # Cannot use a LIMIT for UPDATE or DELETE against a WITHOUT ROWID table + # or a VIEW. (We should fix this someday). + # + db close + sqlite3 db :memory: + do_execsql_test wherelimit-4.1 { + CREATE TABLE t1(a int); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(3); + CREATE TABLE t2(a int); + INSERT INTO t2 SELECT a+100 FROM t1; + CREATE VIEW tv(r,a) AS + SELECT rowid, a FROM t2 UNION ALL SELECT rowid, a FROM t1; + CREATE TRIGGER tv_del INSTEAD OF DELETE ON tv + BEGIN + DELETE FROM t1 WHERE rowid=old.r; + DELETE FROM t2 WHERE rowid=old.r; + END; + } {} + do_catchsql_test wherelimit-4.2 { + DELETE FROM tv WHERE 1 LIMIT 2; + } {1 {ORDER BY and LIMIT not support for table tv}} + do_catchsql_test wherelimit-4.3 { + DELETE FROM tv WHERE 1 ORDER BY a LIMIT 2; + } {1 {ORDER BY and LIMIT not support for table tv}} + do_execsql_test wherelimit-4.10 { + CREATE TABLE t3(a,b,c,d TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t3(a,b,c,d) VALUES(1,2,3,4),(5,6,7,8),(9,10,11,12); + } {} + do_catchsql_test wherelimit-4.11 { + DELETE FROM t3 WHERE a=5 LIMIT 2; + } {1 {ORDER BY and LIMIT not support for table t3}} + do_execsql_test wherelimit-4.12 { + SELECT a,b,c,d FROM t3 ORDER BY 1; + } {1 2 3 4 5 6 7 8 9 10 11 12} + } finish_test From 4ff8431fd161d4d55fed8ba4b88a1c37a4357b76 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 16:30:55 +0000 Subject: [PATCH 107/156] Initial work on porting the changes on this branch to Win32. FossilOrigin-Name: 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 --- manifest | 18 ++-- manifest.uuid | 2 +- src/os_win.c | 201 ++++++++++++++++++++++++++++++++++++------- test/walro2.test | 7 -- test/walrofault.test | 7 -- 5 files changed, 179 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index a22cee602e..dbc8286837 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode. -D 2017-11-08T19:26:27.278 +C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32. +D 2017-11-09T16:30:55.245 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 +F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e -F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d +F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b +F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 -R eba2a72214a6da7d512bda6426cfe5de -U drh -Z 493ea33d970c37f0e10aaccf35cd6d47 +P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 +R a3370fa9345068426fa8edd8924927fc +U mistachkin +Z fbbc6e55aa870fcd83aef8acf0e01375 diff --git a/manifest.uuid b/manifest.uuid index a866afde61..f5c23f13f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 \ No newline at end of file +3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 245a86045b..9bfcfe1c81 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2098,6 +2098,17 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +/* +** The "winIsLockingError" macro is used to determine if a particular I/O +** error code is due to file locking. It must accept the error code DWORD +** as its only argument and should return non-zero if the error code is due +** to file locking. +*/ +#if !defined(winIsLockingError) +#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_IO_PENDING)) +#endif + /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3673,6 +3684,9 @@ struct winShmNode { int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ void *pMap; @@ -3820,6 +3834,116 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } +/* +** Query the status of the DMS lock for the specified file. Returns +** SQLITE_OK upon success. Upon success, the integer pointed to by +** the pLockType argument will be set to the lock type held by the +** other process, as follows: +** +** WINSHM_UNLCK -- No locks are held on the DMS. +** WINSHM_RDLCK -- A SHARED lock is held on the DMS. +** WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS. +*/ +static int winGetShmDmsLockType( + winFile *pFile, /* File handle object */ + int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ +){ +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */ +#endif + LPVOID pOverlapped = 0; + sqlite3_int64 offset = WIN_SHM_DMS; + BYTE notUsed1 = 0; + DWORD notUsed2 = 0; + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( winSeekFile(pFile, offset) ){ + return SQLITE_IOERR_SEEK; + } +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + pOverlapped = &overlapped; +#endif + if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + if( winIsLockingError(osGetLastError()) ){ + *pLockType = WINSHM_WRLCK; + }else{ + return SQLITE_IOERR_READ; + } + }else{ + if( winIsLockingError(osGetLastError()) ){ + *pLockType = WINSHM_RDLCK; + }else{ + return SQLITE_IOERR_WRITE; + } + } + }else{ + *pLockType = WINSHM_UNLCK; + } + return SQLITE_OK; +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){ + int lockType; + int rc = SQLITE_OK; + + /* Use ReadFile/WriteFile to determine the locks other processes are + ** holding on the DMS byte. If it indicates that another process is + ** holding a SHARED lock, then this process may also take a SHARED + ** lock and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){ + rc = SQLITE_IOERR_LOCK; + }else if( lockType==WINSHM_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + }else if( lockType==WINSHM_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK ); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + } + + return rc; +} + /* ** Open the shared-memory area associated with database file pDbFd. ** @@ -3828,11 +3952,12 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ ** the file must be truncated to zero length or have its header cleared. */ static int winOpenSharedMemory(winFile *pDbFd){ - struct winShm *p; /* The connection to be opened */ - struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct winShmNode *pNew; /* Newly allocated winShmNode */ - int nName; /* Size of zName in bytes */ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + int rc2 = SQLITE_ERROR; /* winOpen result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ @@ -3878,30 +4003,29 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } - rc = winOpen(pDbFd->pVfs, - pShmNode->zFilename, /* Name of the file (UTF-8) */ - (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - 0); - if( SQLITE_OK!=rc ){ - goto shm_open_err; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, + 0); + } + if( rc2!=SQLITE_OK ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE, + 0); + if( rc2!=SQLITE_OK ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), + "winOpenShm", pShmNode->zFilename); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); - if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winOpenShm", pDbFd->zPath); - } - } - if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + rc = winLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ @@ -4128,6 +4252,8 @@ static int winShmMap( winFile *pDbFd = (winFile*)fd; winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; int rc = SQLITE_OK; if( !pShm ){ @@ -4138,6 +4264,11 @@ static int winShmMap( pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ @@ -4184,21 +4315,26 @@ static int winShmMap( } pShmNode->aRegion = apNew; + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } + while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, PAGE_READWRITE, nByte, NULL + NULL, protect, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #endif OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", @@ -4208,11 +4344,11 @@ static int winShmMap( int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; #if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFileFromApp(hMap, flags, iOffset - iOffsetShift, szRegion + iOffsetShift ); #else - pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif @@ -4243,6 +4379,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } diff --git a/test/walro2.test b/test/walro2.test index 5a8ce023eb..4b90618fc6 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -18,13 +18,6 @@ source $testdir/lock_common.tcl source $testdir/wal_common.tcl set ::testprefix walro2 -# These tests are only going to work on unix. -# -if {$::tcl_platform(platform) != "unix"} { - finish_test - return -} - # And only if the build is WAL-capable. # ifcapable !wal { diff --git a/test/walrofault.test b/test/walrofault.test index b396be218c..22d4f96bbc 100644 --- a/test/walrofault.test +++ b/test/walrofault.test @@ -17,13 +17,6 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl set ::testprefix walro2 -# These tests are only going to work on unix. -# -if {$::tcl_platform(platform) != "unix"} { - finish_test - return -} - # And only if the build is WAL-capable. # ifcapable !wal { From 4a9ff91852c8bc23a7a1157d05a789b1a1211dd4 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 17:29:04 +0000 Subject: [PATCH 108/156] Make it possible to use OSTRACE for multi-process testing. FossilOrigin-Name: 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index dbc8286837..d11e831d0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32. -D 2017-11-09T16:30:55.245 +C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting. +D 2017-11-09T17:29:04.731 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -459,7 +459,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 -F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c +F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 -R a3370fa9345068426fa8edd8924927fc +P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 +R b588bb75e731a71d38f9b889268c4e35 U mistachkin -Z fbbc6e55aa870fcd83aef8acf0e01375 +Z 3f860c7c58efb09a7e9d6ae32c9bc1c6 diff --git a/manifest.uuid b/manifest.uuid index f5c23f13f2..866cdfe91f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 \ No newline at end of file +0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 9427844e09..ca8d26e4f9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1092,8 +1092,15 @@ void sqlite3DebugPrintf(const char *zFormat, ...){ sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else fprintf(stdout,"%s", zBuf); fflush(stdout); +#endif } #endif From 8a6fa5d41c3188c93a0b3e01d1395ab88e384a82 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 17:29:18 +0000 Subject: [PATCH 109/156] Corrections to the Win32 porting changes on this branch. FossilOrigin-Name: 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d11e831d0c..021c050212 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting. -D 2017-11-09T17:29:04.731 +C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch. +D 2017-11-09T17:29:18.701 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c +F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 -R b588bb75e731a71d38f9b889268c4e35 +P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 +R 83920a29a292bf4d0aadf8e0cc58f982 U mistachkin -Z 3f860c7c58efb09a7e9d6ae32c9bc1c6 +Z f20b1547624a9a7c878450951251f899 diff --git a/manifest.uuid b/manifest.uuid index 866cdfe91f..f22324c47b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 \ No newline at end of file +0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 9bfcfe1c81..d434cfe927 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3895,7 +3895,7 @@ static int winGetShmDmsLockType( ** connection and no other process already holds a lock, return ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ -static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){ +static int winLockSharedMemory(winShmNode *pShmNode){ int lockType; int rc = SQLITE_OK; @@ -4014,7 +4014,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc2 = winOpen(pDbFd->pVfs, pShmNode->zFilename, (sqlite3_file*)&pShmNode->hFile, - SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE, + SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY, 0); if( rc2!=SQLITE_OK ){ rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), @@ -4024,7 +4024,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->isReadonly = 1; } - rc = winLockSharedMemory(pDbFd, pShmNode); + rc = winLockSharedMemory(pShmNode); if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } @@ -4048,7 +4048,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: @@ -4265,7 +4265,7 @@ static int winShmMap( sqlite3_mutex_enter(pShmNode->mutex); if( pShmNode->isUnlocked ){ - rc = winLockSharedMemory(pDbFd, pShmNode); + rc = winLockSharedMemory(pShmNode); if( rc!=SQLITE_OK ) goto shmpage_out; pShmNode->isUnlocked = 0; } From a5150656494e915b675498f8b6b888a336997bec Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 18:21:51 +0000 Subject: [PATCH 110/156] Corrections to Win32 lock detection for SHM files. FossilOrigin-Name: 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 021c050212..eb409a8a6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch. -D 2017-11-09T17:29:18.701 +C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles. +D 2017-11-09T18:21:51.933 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824 +F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 -R 83920a29a292bf4d0aadf8e0cc58f982 +P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 +R e599061360fdeb01478e7f6faea2d0b3 U mistachkin -Z f20b1547624a9a7c878450951251f899 +Z 310f1e71824409858101a715db0b42e2 diff --git a/manifest.uuid b/manifest.uuid index f22324c47b..238938df65 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 \ No newline at end of file +3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d434cfe927..34069351df 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2105,7 +2105,8 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; ** to file locking. */ #if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION) || \ +#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ ((a)==ERROR_IO_PENDING)) #endif @@ -3867,14 +3868,16 @@ static int winGetShmDmsLockType( pOverlapped = &overlapped; #endif if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + DWORD lastErrno = osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - if( winIsLockingError(osGetLastError()) ){ + lastErrno = osGetLastError(); + if( winIsLockingError(lastErrno) ){ *pLockType = WINSHM_WRLCK; }else{ return SQLITE_IOERR_READ; } }else{ - if( winIsLockingError(osGetLastError()) ){ + if( winIsLockingError(lastErrno) ){ *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; From 9f4aad446b5ed8b0006c323f4df2db1e5743bf90 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 18:53:51 +0000 Subject: [PATCH 111/156] Further corrections to read-only SHM file handling on Win32. FossilOrigin-Name: 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 18 +++++++++++------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index eb409a8a6a..0ecda71659 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles. -D 2017-11-09T18:21:51.933 +C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32. +D 2017-11-09T18:53:51.070 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2 +F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 -R e599061360fdeb01478e7f6faea2d0b3 +P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 +R b94c408b8a8461b2b6bed04445c4e88e U mistachkin -Z 310f1e71824409858101a715db0b42e2 +Z 2f572d772e98ad83ecab51d538357fde diff --git a/manifest.uuid b/manifest.uuid index 238938df65..3ece53ff59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 \ No newline at end of file +43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 34069351df..6b0bb3dbd0 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2105,8 +2105,9 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; ** to file locking. */ #if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED) || \ +#define winIsLockingError(a) (((a)==NO_ERROR) || \ ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_HANDLE_EOF) || \ ((a)==ERROR_IO_PENDING)) #endif @@ -3847,6 +3848,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ */ static int winGetShmDmsLockType( winFile *pFile, /* File handle object */ + int bReadOnly, /* Non-zero if the SHM was opened read-only */ int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ ){ #if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) @@ -3867,24 +3869,25 @@ static int winGetShmDmsLockType( overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); pOverlapped = &overlapped; #endif - if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - DWORD lastErrno = osGetLastError(); + if( bReadOnly || + !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ lastErrno = osGetLastError(); if( winIsLockingError(lastErrno) ){ - *pLockType = WINSHM_WRLCK; + if( pLockType ) *pLockType = WINSHM_WRLCK; }else{ return SQLITE_IOERR_READ; } }else{ if( winIsLockingError(lastErrno) ){ - *pLockType = WINSHM_RDLCK; + if( pLockType ) *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; } } }else{ - *pLockType = WINSHM_UNLCK; + if( pLockType ) *pLockType = WINSHM_UNLCK; } return SQLITE_OK; } @@ -3920,7 +3923,8 @@ static int winLockSharedMemory(winShmNode *pShmNode){ ** process might open and use the *-shm file without truncating it. ** And if the *-shm file has been corrupted by a power failure or ** system crash, the database itself may also become corrupt. */ - if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){ + if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly, + &lockType)!=SQLITE_OK ){ rc = SQLITE_IOERR_LOCK; }else if( lockType==WINSHM_UNLCK ){ if( pShmNode->isReadonly ){ From b3c16b899b323c16ef84994359143746a0af2320 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Nov 2017 19:53:06 +0000 Subject: [PATCH 112/156] Add SQLITE_ENABLE_UPDATE_DELETE_LIMIT for views and WITHOUT ROWID tables. FossilOrigin-Name: 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce --- manifest | 26 +++++----- manifest.uuid | 2 +- src/delete.c | 95 +++++++++++++++++++++++++----------- src/parse.y | 7 +-- src/sqliteInt.h | 9 ++-- src/update.c | 26 ++++++++-- test/wherelimit.test | 12 +++-- test/wherelimit2.test | 109 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 232 insertions(+), 54 deletions(-) create mode 100644 test/wherelimit2.test diff --git a/manifest b/manifest index 9a2dbc0bc9..4f63569478 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound. -D 2017-11-09T03:55:09.531 +C Add\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfor\sviews\sand\sWITHOUT\sROWID\stables. +D 2017-11-09T19:53:06.815 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 -F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 +F src/delete.c 001653cc4065ad83d47f4307c29572437f778f1fe249d482894e100275d27efd F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 @@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa +F src/parse.y 5fbc16a4adf5da8ee22a1ef271e001fd6b766eebc3195b52ba2c30cecbbaf2d0 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -475,7 +475,7 @@ F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8 F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e +F src/sqliteInt.h 821479be791acfdb63177b21a491e604526191c1630d1d65d7a4a7c65d203c83 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -535,7 +535,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc -F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0 +F src/update.c 5123fcb4aa98a705ced2acb172c1761a570d142d77901b4f4ce38acb191ef8ed F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 @@ -1557,7 +1557,8 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 -F test/wherelimit.test baa8e5e30b2eddc395cbb3edb313deea14ca6938ea8a96c02a03c34d0ab84b5b +F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8 +F test/wherelimit2.test 501e470c9cf62e96359e3386ae520874792b83322060168ee9c708b2e953b985 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 @@ -1673,7 +1674,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e -R a028f3be0864f739aeb3787daf46fc8e -U drh -Z 97937a65b1ac3d4b961579eac7d25cbd +P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 +R 46792a4c59e5e3fb96fc57893e581fbe +T *branch * update-delete-limit-fix +T *sym-update-delete-limit-fix * +T -sym-trunk * +U dan +Z c86d100b98964de2c810aa6a7710c91c diff --git a/manifest.uuid b/manifest.uuid index e270b8e48e..e7ae4dab5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 \ No newline at end of file +584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index cd78f68f41..509fd8af8d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -90,6 +90,9 @@ void sqlite3MaterializeView( Parse *pParse, /* Parsing context */ Table *pView, /* View definition */ Expr *pWhere, /* Optional WHERE clause to be added */ + ExprList *pOrderBy, /* Optional ORDER BY clause */ + Expr *pLimit, /* Optional LIMIT clause */ + Expr *pOffset, /* Optional OFFSET clause */ int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; @@ -106,8 +109,8 @@ void sqlite3MaterializeView( assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, - SF_IncludeHidden, 0, 0); + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, + SF_IncludeHidden, pLimit, pOffset); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -132,18 +135,23 @@ Expr *sqlite3LimitWhere( Expr *pOffset, /* The OFFSET clause. May be null */ char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ ){ - Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + sqlite3 *db = pParse->db; + Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - Expr *pSelectRowid = NULL; /* SELECT rowid ... */ ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ Select *pSelect = NULL; /* Complete SELECT tree */ + Table *pTab; /* Check that there isn't an ORDER BY without a LIMIT clause. */ - if( pOrderBy && (pLimit == 0) ) { + if( pOrderBy && pLimit==0 ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup; + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); + sqlite3ExprDelete(pParse->db, pOffset); + return 0; } /* We only need to generate a select expression if there @@ -164,36 +172,44 @@ Expr *sqlite3LimitWhere( ** ); */ - pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup; - pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup; + pTab = pSrc->a[0].pTab; + if( HasRowid(pTab) ){ + pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); + pEList = sqlite3ExprListAppend( + pParse, 0, sqlite3PExpr(pParse, TK_ROW, 0, 0) + ); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + if( pPk->nKeyCol==1 ){ + pLhs = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[0]].zName); + }else{ + int i; + for(i=0; inKeyCol; i++){ + Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zName); + pEList = sqlite3ExprListAppend(pParse, pEList, p); + } + pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( pLhs ){ + pLhs->x.pList = sqlite3ExprListDup(db, pEList, 0); + } + } + } /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ + pSrc->a[0].pTab = 0; pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); - if( pSelectSrc == 0 ) { - sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup; - } + pSrc->a[0].pTab = pTab; /* generate the SELECT expression tree. */ - pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0, - pOrderBy,0,pLimit,pOffset); - if( pSelect == 0 ) return 0; + pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, + pOrderBy,0,pLimit,pOffset + ); /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ - pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0); - pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0) : 0; + pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; - -limit_where_cleanup: - sqlite3ExprDelete(pParse->db, pWhere); - sqlite3ExprListDelete(pParse->db, pOrderBy); - sqlite3ExprDelete(pParse->db, pLimit); - sqlite3ExprDelete(pParse->db, pOffset); - return 0; } #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ /* && !defined(SQLITE_OMIT_SUBQUERY) */ @@ -205,10 +221,13 @@ limit_where_cleanup: ** \________/ \________________/ ** pTabList pWhere */ -void sqlite3DeleteFrom( +void sqlite3DeleteFromLimit( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table from which we should delete things */ - Expr *pWhere /* The WHERE clause. May be null */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, + Expr *pLimit, + Expr *pOffset ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ @@ -253,6 +272,7 @@ void sqlite3DeleteFrom( } assert( pTabList->nSrc==1 ); + /* Locate the table which we want to delete. This table has to be ** put in an SrcList structure because some of the subroutines we ** will be calling are designed to work with multiple tables and expect @@ -277,6 +297,16 @@ void sqlite3DeleteFrom( # define isView 0 #endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "DELETE" + ); + pOrderBy = 0; + pLimit = pOffset = 0; + } +#endif + /* If pTab is really a view, make sure it has been initialized. */ if( sqlite3ViewGetColumnNames(pParse, pTab) ){ @@ -324,8 +354,12 @@ void sqlite3DeleteFrom( */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur); + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, pOffset, iTabCur + ); iDataCur = iIdxCur = iTabCur; + pOrderBy = 0; + pLimit = pOffset = 0; } #endif @@ -569,6 +603,9 @@ delete_from_cleanup: sqlite3AuthContextPop(&sContext); sqlite3SrcListDelete(db, pTabList); sqlite3ExprDelete(db, pWhere); + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); + sqlite3ExprDelete(db, pOffset); sqlite3DbFree(db, aToOpen); return; } diff --git a/src/parse.y b/src/parse.y index a8d080eed3..928e53dd82 100644 --- a/src/parse.y +++ b/src/parse.y @@ -753,8 +753,7 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); - W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); - sqlite3DeleteFrom(pParse,X,W); + sqlite3DeleteFromLimit(pParse,X,W,O,L.pLimit,L.pOffset); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT @@ -779,8 +778,10 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); +#if 0 W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); - sqlite3Update(pParse,X,Y,W,R); +#endif + sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 378b085021..d3c412e222 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3764,8 +3764,11 @@ void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif -void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); -void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); +void sqlite3DeleteFromLimit(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*); +#define sqlite3DeleteFrom(x,y,z) sqlite3DeleteFromLimit(x,y,z,0,0,0) +void sqlite3UpdateLimit(Parse*, SrcList*, ExprList*, Expr*, int, + ExprList*,Expr*,Expr*); +#define sqlite3Update(v,w,x,y,z) sqlite3UpdateLimit(v,w,x,y,z,0,0,0) WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); LogEst sqlite3WhereOutputRowCount(WhereInfo*); @@ -3889,7 +3892,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3*); void sqlite3ChangeCookie(Parse*, int); #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) -void sqlite3MaterializeView(Parse*, Table*, Expr*, int); +void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,Expr*,int); #endif #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/update.c b/src/update.c index e69efdb6bf..f63a9ffe8a 100644 --- a/src/update.c +++ b/src/update.c @@ -86,12 +86,15 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ ** \_______/ \________/ \______/ \________________/ * onError pTabList pChanges pWhere */ -void sqlite3Update( +void sqlite3UpdateLimit( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table in which we should change things */ ExprList *pChanges, /* Things to be changed */ Expr *pWhere, /* The WHERE clause. May be null */ - int onError /* How to handle constraint errors */ + int onError, /* How to handle constraint errors */ + ExprList *pOrderBy, + Expr *pLimit, + Expr *pOffset ){ int i, j; /* Loop counters */ Table *pTab; /* The table to be updated */ @@ -176,6 +179,16 @@ void sqlite3Update( # define isView 0 #endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "UPDATE" + ); + pOrderBy = 0; + pLimit = pOffset = 0; + } +#endif + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } @@ -344,7 +357,11 @@ void sqlite3Update( */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iDataCur); + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, pOffset, iDataCur + ); + pOrderBy = 0; + pLimit = pOffset = 0; } #endif @@ -728,6 +745,9 @@ update_cleanup: sqlite3SrcListDelete(db, pTabList); sqlite3ExprListDelete(db, pChanges); sqlite3ExprDelete(db, pWhere); + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); + sqlite3ExprDelete(db, pOffset); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise diff --git a/test/wherelimit.test b/test/wherelimit.test index e10d90b40b..e2ba18d2c8 100644 --- a/test/wherelimit.test +++ b/test/wherelimit.test @@ -38,6 +38,8 @@ proc create_test_data {size} { ifcapable {update_delete_limit} { + execsql { CREATE TABLE t1(x, y) } + # check syntax error support do_test wherelimit-0.1 { catchsql {DELETE FROM t1 ORDER BY x} @@ -49,6 +51,8 @@ ifcapable {update_delete_limit} { catchsql {UPDATE t1 SET y=1 WHERE x=1 ORDER BY x} } {1 {ORDER BY without LIMIT on UPDATE}} + execsql { DROP TABLE t1 } + # no AS on table sources do_test wherelimit-0.4 { catchsql {DELETE FROM t1 AS a WHERE x=1} @@ -301,20 +305,20 @@ ifcapable {update_delete_limit} { } {} do_catchsql_test wherelimit-4.2 { DELETE FROM tv WHERE 1 LIMIT 2; - } {1 {ORDER BY and LIMIT not support for table tv}} + } {0 {}} do_catchsql_test wherelimit-4.3 { DELETE FROM tv WHERE 1 ORDER BY a LIMIT 2; - } {1 {ORDER BY and LIMIT not support for table tv}} + } {0 {}} do_execsql_test wherelimit-4.10 { CREATE TABLE t3(a,b,c,d TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; INSERT INTO t3(a,b,c,d) VALUES(1,2,3,4),(5,6,7,8),(9,10,11,12); } {} do_catchsql_test wherelimit-4.11 { DELETE FROM t3 WHERE a=5 LIMIT 2; - } {1 {ORDER BY and LIMIT not support for table t3}} + } {0 {}} do_execsql_test wherelimit-4.12 { SELECT a,b,c,d FROM t3 ORDER BY 1; - } {1 2 3 4 5 6 7 8 9 10 11 12} + } {1 2 3 4 9 10 11 12} } diff --git a/test/wherelimit2.test b/test/wherelimit2.test new file mode 100644 index 0000000000..e505da42bb --- /dev/null +++ b/test/wherelimit2.test @@ -0,0 +1,109 @@ +# 2008 October 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the LIMIT ... OFFSET ... clause +# of UPDATE and DELETE statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix wherelimit2 + +ifcapable !update_delete_limit { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 'f'); + INSERT INTO t1 VALUES(2, 'e'); + INSERT INTO t1 VALUES(3, 'd'); + INSERT INTO t1 VALUES(4, 'c'); + INSERT INTO t1 VALUES(5, 'b'); + INSERT INTO t1 VALUES(6, 'a'); + + CREATE VIEW v1 AS SELECT a,b FROM t1; + CREATE TABLE log(op, a); + + CREATE TRIGGER v1del INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete', old.a); + END; + + CREATE TRIGGER v1upd INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update', old.a); + END; +} + +do_execsql_test 1.1 { + DELETE FROM v1 ORDER BY a LIMIT 3; + SELECT * FROM log; DELETE FROM log; +} { + delete 1 delete 2 delete 3 +} +do_execsql_test 1.2 { + DELETE FROM v1 ORDER BY b LIMIT 3; + SELECT * FROM log; DELETE FROM log; +} { + delete 6 delete 5 delete 4 +} +do_execsql_test 1.3 { + UPDATE v1 SET b = 555 ORDER BY a LIMIT 3; + SELECT * FROM log; DELETE FROM log; +} { + update 1 update 2 update 3 +} +do_execsql_test 1.4 { + UPDATE v1 SET b = 555 ORDER BY b LIMIT 3; + SELECT * FROM log; DELETE FROM log; +} { + update 6 update 5 update 4 +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 1, 'h'); + INSERT INTO t2 VALUES(1, 2, 'g'); + INSERT INTO t2 VALUES(2, 1, 'f'); + INSERT INTO t2 VALUES(2, 2, 'e'); + INSERT INTO t2 VALUES(3, 1, 'd'); + INSERT INTO t2 VALUES(3, 2, 'c'); + INSERT INTO t2 VALUES(4, 1, 'b'); + INSERT INTO t2 VALUES(4, 2, 'a'); +} + +do_execsql_test 2.1 { + BEGIN; + DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2; + SELECT c FROM t2 ORDER BY 1; + ROLLBACK; +} {a c e f g h} + +do_execsql_test 2.2 { + BEGIN; + UPDATE t2 SET c=NULL ORDER BY a, b DESC LIMIT 3 OFFSET 1; + SELECT a, b, c FROM t2; + ROLLBACK; +} { + 1 1 {} + 1 2 g + 2 1 {} + 2 2 {} + 3 1 d + 3 2 c + 4 1 b + 4 2 a +} + + + +finish_test + From bcb416a9ff954bbc22bd226615edc335a6989666 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 20:02:44 +0000 Subject: [PATCH 113/156] Get read-only SHM file tests passing on Win32. FossilOrigin-Name: abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 32 +++++++++++++++++++++----------- test/walro2.test | 2 ++ test/walrofault.test | 1 + 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 0ecda71659..05976da356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32. -D 2017-11-09T18:53:51.070 +C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32. +D 2017-11-09T20:02:44.785 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4 +F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b -F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de +F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00 +F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 -R b94c408b8a8461b2b6bed04445c4e88e +P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce +R b51af4203460b2806284a868a4615565 U mistachkin -Z 2f572d772e98ad83ecab51d538357fde +Z 2f0a43c33871392e0df8028f39e6b334 diff --git a/manifest.uuid b/manifest.uuid index 3ece53ff59..dd49a781d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce \ No newline at end of file +abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6b0bb3dbd0..05ae7e67db 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2099,18 +2099,26 @@ static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; /* -** The "winIsLockingError" macro is used to determine if a particular I/O -** error code is due to file locking. It must accept the error code DWORD -** as its only argument and should return non-zero if the error code is due -** to file locking. +** The "winIsLockConflict" macro is used to determine if a particular I/O +** error code is due to a file locking conflict. It must accept the error +** code DWORD as its only argument. */ -#if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==NO_ERROR) || \ +#if !defined(winIsLockConflict) +#define winIsLockConflict(a) (((a)==NO_ERROR) || \ ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_HANDLE_EOF) || \ ((a)==ERROR_IO_PENDING)) #endif +/* +** The "winIsLockMissing" macro is used to determine if a particular I/O +** error code is due to being unable to obtain a file lock because all or +** part of the range requested within the file is missing. It must accept +** the error code DWORD as its only argument. +*/ +#if !defined(winIsLockMissing) +#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF)) +#endif + /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3874,13 +3882,15 @@ static int winGetShmDmsLockType( DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ lastErrno = osGetLastError(); - if( winIsLockingError(lastErrno) ){ + if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_WRLCK; + }else if( winIsLockMissing(lastErrno) ){ + if( pLockType ) *pLockType = WINSHM_UNLCK; }else{ return SQLITE_IOERR_READ; } }else{ - if( winIsLockingError(lastErrno) ){ + if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; @@ -4024,8 +4034,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY, 0); if( rc2!=SQLITE_OK ){ - rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), - "winOpenShm", pShmNode->zFilename); + rc = winLogError(rc2, osGetLastError(), "winOpenShm", + pShmNode->zFilename); goto shm_open_err; } pShmNode->isReadonly = 1; diff --git a/test/walro2.test b/test/walro2.test index 4b90618fc6..67023de1a2 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -255,6 +255,8 @@ do_multiclient_test tn { file size test.db-wal } {461152} do_test 4.2.4 { + file_control_persist_wal db 1; db close + forcecopy test.db test.db2 forcecopy test.db-wal test.db2-wal forcecopy test.db-shm test.db2-shm diff --git a/test/walrofault.test b/test/walrofault.test index 22d4f96bbc..3e66e2d920 100644 --- a/test/walrofault.test +++ b/test/walrofault.test @@ -42,6 +42,7 @@ do_execsql_test 1.0 { WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) INSERT INTO t1(b) SELECT randomblob(800) FROM s; } {wal} +file_control_persist_wal db 1; db close faultsim_save_and_close do_faultsim_test 1 -faults oom* -prep { From e94187780620d5c1b6ceafe3e90ac15f17fe428b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Nov 2017 20:34:35 +0000 Subject: [PATCH 114/156] Enhance walro2.test to better ensure that readonly_shm clients are not using invalid *-shm files. FossilOrigin-Name: ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c --- manifest | 14 +++--- manifest.uuid | 2 +- test/walro2.test | 110 ++++++++++++++++++++++++----------------------- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 05976da356..e214aee04c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32. -D 2017-11-09T20:02:44.785 +C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles. +D 2017-11-09T20:34:35.406 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00 +F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce -R b51af4203460b2806284a868a4615565 -U mistachkin -Z 2f0a43c33871392e0df8028f39e6b334 +P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 +R 43c8def15803418ff2b6211daf889aff +U dan +Z b374f730e5d2822de9c4c678f7c53c0c diff --git a/manifest.uuid b/manifest.uuid index dd49a781d5..d156b1b802 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 \ No newline at end of file +ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c \ No newline at end of file diff --git a/test/walro2.test b/test/walro2.test index 67023de1a2..9d0a6b4a19 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -25,6 +25,22 @@ ifcapable !wal { return } +proc copy_to_test2 {bZeroShm} { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + if {$bZeroShm} { + forcedelete test.db2-shm + set fd [open test.db2-shm w] + seek $fd [expr [file size test.db-shm]-1] + puts -nonewline $fd "\0" + close $fd + } else { + forcecopy test.db-shm test.db2-shm + } +} + +foreach bZeroShm {0 1} { +set TN [expr $bZeroShm+1] do_multiclient_test tn { # Close all connections and delete the database. @@ -45,7 +61,7 @@ do_multiclient_test tn { } } - do_test 1.1 { + do_test $TN.1.1 { code2 { sqlite3 db2 test.db } sql2 { CREATE TABLE t1(x, y); @@ -56,26 +72,24 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 1.2.1 { - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + do_test $TN.1.2.1 { + copy_to_test2 $bZeroShm code1 { sqlite3 db file:test.db2?readonly_shm=1 } sql1 { SELECT * FROM t1 } } {a b c d} - do_test 1.2.2 { + do_test $TN.1.2.2 { sql1 { SELECT * FROM t1 } } {a b c d} - do_test 1.3.1 { + do_test $TN.1.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d} - do_test 1.3.2 { + do_test $TN.1.3.2 { sql1 { SELECT * FROM t1 } } {a b c d} @@ -83,7 +97,7 @@ do_multiclient_test tn { code2 { db2 close } code3 { db3 close } - do_test 2.1 { + do_test $TN.2.1 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES('e', 'f'); @@ -92,10 +106,8 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 2.2 { - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + do_test $TN.2.2 { + copy_to_test2 $bZeroShm code1 { sqlite3 db file:test.db2?readonly_shm=1 } @@ -105,16 +117,16 @@ do_multiclient_test tn { } } {a b c d e f g h} - do_test 2.3.1 { + do_test $TN.2.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d e f g h} - do_test 2.3.2 { + do_test $TN.2.3.2 { sql3 { INSERT INTO t1 VALUES('i', 'j') } code3 { db3 close } sql1 { COMMIT } } {} - do_test 2.3.3 { + do_test $TN.2.3.3 { sql1 { SELECT * FROM t1 } } {a b c d e f g h i j} @@ -134,10 +146,10 @@ do_multiclient_test tn { catch { code1 { db close } } catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 3.1.0 { + do_test $TN.3.1.0 { list [file exists test.db-wal] [file exists test.db-shm] } {0 0} - do_test 3.1.1 { + do_test $TN.3.1.1 { close [open test.db-wal w] close [open test.db-shm w] code1 { @@ -146,20 +158,20 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {a b c d e f g h} - do_test 3.2.0 { + do_test $TN.3.2.0 { list [file size test.db-wal] [file size test.db-shm] } {0 0} - do_test 3.2.1 { + do_test $TN.3.2.1 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate } code2 { db2 close } sql1 { SELECT * FROM t1 } } {a b c d e f g h 1 2} - do_test 3.2.2 { + do_test $TN.3.2.2 { list [file size test.db-wal] [file size test.db-shm] } {0 32768} - do_test 3.3.0 { + do_test $TN.3.3.0 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES(3, 4); @@ -171,11 +183,11 @@ do_multiclient_test tn { code1 { db close } list [file size test.db-wal] [file size test.db-shm] } [list [wal_file_size 4 1024] 32768] - do_test 3.3.1 { + do_test $TN.3.3.1 { code1 { sqlite3 db file:test.db?readonly_shm=1 } sql1 { SELECT * FROM t1 } } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10} - do_test 3.3.2 { + do_test $TN.3.3.2 { code2 { sqlite3 db2 test.db } sql2 { PRAGMA wal_checkpoint; @@ -185,7 +197,7 @@ do_multiclient_test tn { code2 { db2 close } list [file size test.db-wal] [file size test.db-shm] } [list [wal_file_size 4 1024] 32768] - do_test 3.3.3 { + do_test $TN.3.3.3 { sql1 { SELECT * FROM t1 } } {i ii} @@ -196,7 +208,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 4.0 { + do_test $TN.4.0 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -206,19 +218,17 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('world'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm - + copy_to_test2 $bZeroShm + code1 { db close } } {} - do_test 4.1.1 { + do_test $TN.4.1.1 { code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1 } } {hello world} - do_test 4.1.2 { + do_test $TN.4.1.2 { code3 { sqlite3 db3 test.db2 } sql3 { INSERT INTO t1 VALUES('!'); @@ -226,7 +236,7 @@ do_multiclient_test tn { } code3 { db3 close } } {} - do_test 4.1.3 { + do_test $TN.4.1.3 { sql2 { SELECT * FROM t1 } } {hello world !} @@ -234,7 +244,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 4.2.1 { + do_test $TN.4.2.1 { code1 { sqlite3 db test.db } sql1 { INSERT INTO t1 VALUES('!'); @@ -251,16 +261,13 @@ do_multiclient_test tn { SELECT count(*) FROM t2; } } {500} - do_test 4.2.2 { + do_test $TN.4.2.2 { file size test.db-wal } {461152} - do_test 4.2.4 { + do_test $TN.4.2.4 { file_control_persist_wal db 1; db close - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm - + copy_to_test2 $bZeroShm code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1; @@ -275,7 +282,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 5.0 { + do_test $TN.5.0 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -288,21 +295,19 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('hello'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + copy_to_test2 $bZeroShm code1 { db close } } {} - do_test 5.1 { + do_test $TN.5.1 { code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1; } } {hello world ! world hello} - do_test 5.2 { + do_test $TN.5.2 { code1 { proc handle_read {op args} { if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { @@ -325,7 +330,7 @@ do_multiclient_test tn { code1 { set ::res2 } } {hello world ! world hello} - do_test 5.3 { + do_test $TN.5.3 { code1 { db close } code1 { tvfs delete } } {} @@ -337,7 +342,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 6.1 { + do_test $TN.6.1 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -350,14 +355,12 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('hello'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + copy_to_test2 $bZeroShm code1 { db close } } {} - do_test 6.2 { + do_test $TN.6.2 { code1 { set ::nRem 5 proc handle_read {op args} { @@ -380,10 +383,11 @@ do_multiclient_test tn { } } {hello world ! world hello} - do_test 6.3 { + do_test $TN.6.3 { code1 { db close } code1 { tvfs delete } } {} } +} ;# foreach bZeroShm finish_test From 95a05aae679bb79bd697ab082fc6e6b50356f2c5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 20:37:37 +0000 Subject: [PATCH 115/156] Add an assert() in the Win32 VFS. FossilOrigin-Name: 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e214aee04c..68233db669 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles. -D 2017-11-09T20:34:35.406 +C Add\san\sassert()\sin\sthe\sWin32\sVFS. +D 2017-11-09T20:37:37.876 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c +F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 -R 43c8def15803418ff2b6211daf889aff -U dan -Z b374f730e5d2822de9c4c678f7c53c0c +P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c +R 9f94324b020c3efbc23aa89a0c6b1ac2 +U mistachkin +Z b9496fb62e9a99521b89e7f9003a99d8 diff --git a/manifest.uuid b/manifest.uuid index d156b1b802..da346528d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c \ No newline at end of file +22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 05ae7e67db..aff48e25e9 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3885,6 +3885,7 @@ static int winGetShmDmsLockType( if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_WRLCK; }else if( winIsLockMissing(lastErrno) ){ + assert( bReadOnly ); if( pLockType ) *pLockType = WINSHM_UNLCK; }else{ return SQLITE_IOERR_READ; From 7b7f224c6242f032e2e1d84e4aeb78ede1cbcae1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 22:23:50 +0000 Subject: [PATCH 116/156] Revise and vastly simplify the Win32 SHM file locking semantics, allowing all new tests to pass. FossilOrigin-Name: d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 --- manifest | 12 ++--- manifest.uuid | 2 +- src/os_win.c | 128 ++++---------------------------------------------- 3 files changed, 15 insertions(+), 127 deletions(-) diff --git a/manifest b/manifest index 68233db669..bf3dc40e6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sin\sthe\sWin32\sVFS. -D 2017-11-09T20:37:37.876 +C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass. +D 2017-11-09T22:23:50.758 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3 +F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c -R 9f94324b020c3efbc23aa89a0c6b1ac2 +P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 +R 0c5d22f8af57ea73c927075712f52c29 U mistachkin -Z b9496fb62e9a99521b89e7f9003a99d8 +Z e19a1ec884226a6dec0cb1fab52ceb52 diff --git a/manifest.uuid b/manifest.uuid index da346528d8..2503e60869 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 \ No newline at end of file +d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index aff48e25e9..975df6154b 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2098,27 +2098,6 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; -/* -** The "winIsLockConflict" macro is used to determine if a particular I/O -** error code is due to a file locking conflict. It must accept the error -** code DWORD as its only argument. -*/ -#if !defined(winIsLockConflict) -#define winIsLockConflict(a) (((a)==NO_ERROR) || \ - ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_IO_PENDING)) -#endif - -/* -** The "winIsLockMissing" macro is used to determine if a particular I/O -** error code is due to being unable to obtain a file lock because all or -** part of the range requested within the file is missing. It must accept -** the error code DWORD as its only argument. -*/ -#if !defined(winIsLockMissing) -#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF)) -#endif - /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3844,65 +3823,6 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } -/* -** Query the status of the DMS lock for the specified file. Returns -** SQLITE_OK upon success. Upon success, the integer pointed to by -** the pLockType argument will be set to the lock type held by the -** other process, as follows: -** -** WINSHM_UNLCK -- No locks are held on the DMS. -** WINSHM_RDLCK -- A SHARED lock is held on the DMS. -** WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS. -*/ -static int winGetShmDmsLockType( - winFile *pFile, /* File handle object */ - int bReadOnly, /* Non-zero if the SHM was opened read-only */ - int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ -){ -#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) - OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */ -#endif - LPVOID pOverlapped = 0; - sqlite3_int64 offset = WIN_SHM_DMS; - BYTE notUsed1 = 0; - DWORD notUsed2 = 0; - -#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) - if( winSeekFile(pFile, offset) ){ - return SQLITE_IOERR_SEEK; - } -#else - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(offset & 0xffffffff); - overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); - pOverlapped = &overlapped; -#endif - if( bReadOnly || - !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); - if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - lastErrno = osGetLastError(); - if( winIsLockConflict(lastErrno) ){ - if( pLockType ) *pLockType = WINSHM_WRLCK; - }else if( winIsLockMissing(lastErrno) ){ - assert( bReadOnly ); - if( pLockType ) *pLockType = WINSHM_UNLCK; - }else{ - return SQLITE_IOERR_READ; - } - }else{ - if( winIsLockConflict(lastErrno) ){ - if( pLockType ) *pLockType = WINSHM_RDLCK; - }else{ - return SQLITE_IOERR_WRITE; - } - } - }else{ - if( pLockType ) *pLockType = WINSHM_UNLCK; - } - return SQLITE_OK; -} - /* ** The DMS lock has not yet been taken on shm file pShmNode. Attempt to ** take it now. Return SQLITE_OK if successful, or an SQLite error @@ -3913,53 +3833,21 @@ static int winGetShmDmsLockType( ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ static int winLockSharedMemory(winShmNode *pShmNode){ - int lockType; - int rc = SQLITE_OK; - - /* Use ReadFile/WriteFile to determine the locks other processes are - ** holding on the DMS byte. If it indicates that another process is - ** holding a SHARED lock, then this process may also take a SHARED - ** lock and proceed with opening the *-shm file. - ** - ** Or, if no other process is holding any lock, then this process - ** is the first to open it. In this case take an EXCLUSIVE lock on the - ** DMS byte and truncate the *-shm file to zero bytes in size. Then - ** downgrade to a SHARED lock on the DMS byte. - ** - ** If another process is holding an EXCLUSIVE lock on the DMS byte, - ** return SQLITE_BUSY to the caller (it will try again). An earlier - ** version of this code attempted the SHARED lock at this point. But - ** this introduced a subtle race condition: if the process holding - ** EXCLUSIVE failed just before truncating the *-shm file, then this - ** process might open and use the *-shm file without truncating it. - ** And if the *-shm file has been corrupted by a power failure or - ** system crash, the database itself may also become corrupt. */ - if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly, - &lockType)!=SQLITE_OK ){ - rc = SQLITE_IOERR_LOCK; - }else if( lockType==WINSHM_UNLCK ){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; - rc = SQLITE_READONLY_CANTINIT; - }else{ winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); - if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), "winLockSharedMemory", pShmNode->zFilename); - } } - }else if( lockType==WINSHM_WRLCK ){ - rc = SQLITE_BUSY; } - if( rc==SQLITE_OK ){ - assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK ); - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); - } - - return rc; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } /* From 10813713baec7d78f700c19e51107a19bb698d98 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 22:25:58 +0000 Subject: [PATCH 117/156] Cleanup superfluous whitespace changes. FossilOrigin-Name: a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index bf3dc40e6b..125e07f68d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass. -D 2017-11-09T22:23:50.758 +C Cleanup\ssuperfluous\swhitespace\schanges. +D 2017-11-09T22:25:58.319 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88 +F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 -R 0c5d22f8af57ea73c927075712f52c29 +P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 +R d62fe3d1eed9309359251a90225ada34 U mistachkin -Z e19a1ec884226a6dec0cb1fab52ceb52 +Z 112940bd575d10def1c3f99e15024bb0 diff --git a/manifest.uuid b/manifest.uuid index 2503e60869..1dbf5a7d5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 \ No newline at end of file +a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 975df6154b..2ee4ad1acf 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3858,12 +3858,12 @@ static int winLockSharedMemory(winShmNode *pShmNode){ ** the file must be truncated to zero length or have its header cleared. */ static int winOpenSharedMemory(winFile *pDbFd){ - struct winShm *p; /* The connection to be opened */ - winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc = SQLITE_OK; /* Result code */ - int rc2 = SQLITE_ERROR; /* winOpen result code */ - winShmNode *pNew; /* Newly allocated winShmNode */ - int nName; /* Size of zName in bytes */ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + int rc2 = SQLITE_ERROR; /* winOpen result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ From 0e026f403b9900351cfa11c67f9a4682f2911381 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 23:24:29 +0000 Subject: [PATCH 118/156] Avoid superfluous SHM unlock call in the Win32 VFS. FossilOrigin-Name: 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 125e07f68d..453bed54e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanup\ssuperfluous\swhitespace\schanges. -D 2017-11-09T22:25:58.319 +C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS. +D 2017-11-09T23:24:29.716 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0 +F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 -R d62fe3d1eed9309359251a90225ada34 +P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed +R 9d6be609689f3743af6adbbc5cf401b2 U mistachkin -Z 112940bd575d10def1c3f99e15024bb0 +Z 39f42e393c5183ce2049496b416ae681 diff --git a/manifest.uuid b/manifest.uuid index 1dbf5a7d5c..ba222182ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed \ No newline at end of file +5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2ee4ad1acf..086bbf90aa 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3834,6 +3834,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ */ static int winLockSharedMemory(winShmNode *pShmNode){ int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; @@ -3846,7 +3847,10 @@ static int winLockSharedMemory(winShmNode *pShmNode){ } } - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } From aca84e6a8bb48f407b67994e0af93f7bf9099112 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 10 Nov 2017 12:41:21 +0000 Subject: [PATCH 119/156] Fix harmless compiler warning seen with MSVC. FossilOrigin-Name: 3711ef2366af8fefccaaa0a6ee520ce6bc9c74a4fe0666f0a85ef96be46e02d3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/dbpage.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9a2dbc0bc9..20d7f8ee03 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound. -D 2017-11-09T03:55:09.531 +C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. +D 2017-11-10T12:41:21.768 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -420,7 +420,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde +F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e -R a028f3be0864f739aeb3787daf46fc8e -U drh -Z 97937a65b1ac3d4b961579eac7d25cbd +P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 +R 480986b4349d5fc99d8b880800c9dbc1 +U mistachkin +Z 0f581c91a91b83b96a80c021b9d15d0e diff --git a/manifest.uuid b/manifest.uuid index e270b8e48e..476c0bbc49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 \ No newline at end of file +3711ef2366af8fefccaaa0a6ee520ce6bc9c74a4fe0666f0a85ef96be46e02d3 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 75e4c42e67..c38de3b39f 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -303,7 +303,7 @@ static int dbpageUpdate( sqlite_int64 *pRowid ){ DbpageTable *pTab = (DbpageTable *)pVtab; - int pgno; + Pgno pgno; DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; @@ -318,7 +318,7 @@ static int dbpageUpdate( goto update_fail; } pgno = sqlite3_value_int(argv[0]); - if( sqlite3_value_int(argv[1])!=pgno ){ + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ zErr = "cannot insert"; goto update_fail; } From 26caf5bef16bf48e2ea026104807b23f83c95dab Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Nov 2017 15:42:21 +0000 Subject: [PATCH 120/156] Fix a problem with (DELETE...LIMIT) statements against WITHOUT ROWID tables with a single column PK. FossilOrigin-Name: 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc --- manifest | 19 ++++----- manifest.uuid | 2 +- src/delete.c | 4 +- src/parse.y | 3 -- test/wherelimit2.test | 89 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 4f63569478..72f23a851b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfor\sviews\sand\sWITHOUT\sROWID\stables. -D 2017-11-09T19:53:06.815 +C Fix\sa\sproblem\swith\s(DELETE...LIMIT)\sstatements\sagainst\sWITHOUT\sROWID\stables\nwith\sa\ssingle\scolumn\sPK. +D 2017-11-10T15:42:21.918 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 -F src/delete.c 001653cc4065ad83d47f4307c29572437f778f1fe249d482894e100275d27efd +F src/delete.c cd86beec4f64491270f52521a45f0d335e29766b165511d025ee11119839375a F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 @@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 5fbc16a4adf5da8ee22a1ef271e001fd6b766eebc3195b52ba2c30cecbbaf2d0 +F src/parse.y 49eb13a590f88a03e81f1bc8df0634e56598ebd8d76d38dda24ba9edd7f92194 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -1558,7 +1558,7 @@ F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8 -F test/wherelimit2.test 501e470c9cf62e96359e3386ae520874792b83322060168ee9c708b2e953b985 +F test/wherelimit2.test 565fc74c3f96c3f08f3c20e233530e4c5532e6dfa01d1acc8dd2da5009a3dfd3 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 @@ -1674,10 +1674,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70 -R 46792a4c59e5e3fb96fc57893e581fbe -T *branch * update-delete-limit-fix -T *sym-update-delete-limit-fix * -T -sym-trunk * +P 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce +R d15b259ca771d410cbcab13660cb7826 U dan -Z c86d100b98964de2c810aa6a7710c91c +Z 4ca2b1e389083d1c64078c78c67450a8 diff --git a/manifest.uuid b/manifest.uuid index e7ae4dab5c..ebde4c85ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce \ No newline at end of file +35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 509fd8af8d..34c5feb8b7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -181,7 +181,9 @@ Expr *sqlite3LimitWhere( }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); if( pPk->nKeyCol==1 ){ - pLhs = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[0]].zName); + const char *zName = pTab->aCol[pPk->aiColumn[0]].zName; + pLhs = sqlite3Expr(db, TK_ID, zName); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); }else{ int i; for(i=0; inKeyCol; i++){ diff --git a/src/parse.y b/src/parse.y index 928e53dd82..2a536b0a8e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -778,9 +778,6 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); -#if 0 - W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); -#endif sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset); } %endif diff --git a/test/wherelimit2.test b/test/wherelimit2.test index e505da42bb..1bf0168b50 100644 --- a/test/wherelimit2.test +++ b/test/wherelimit2.test @@ -22,6 +22,9 @@ ifcapable !update_delete_limit { return } +#------------------------------------------------------------------------- +# Test with views and INSTEAD OF triggers. +# do_execsql_test 1.0 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 'f'); @@ -68,6 +71,9 @@ do_execsql_test 1.4 { update 6 update 5 update 4 } +#------------------------------------------------------------------------- +# Simple test using WITHOUT ROWID table. +# do_execsql_test 2.0 { CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; INSERT INTO t2 VALUES(1, 1, 'h'); @@ -103,6 +109,89 @@ do_execsql_test 2.2 { 4 2 a } +#------------------------------------------------------------------------- +# Test using a virtual table +# +ifcapable fts5 { + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft(rowid, x) VALUES(-45, 'a a'); + INSERT INTO ft(rowid, x) VALUES(12, 'a b'); + INSERT INTO ft(rowid, x) VALUES(444, 'a c'); + INSERT INTO ft(rowid, x) VALUES(12300, 'a d'); + INSERT INTO ft(rowid, x) VALUES(25400, 'a c'); + INSERT INTO ft(rowid, x) VALUES(25401, 'a b'); + INSERT INTO ft(rowid, x) VALUES(50000, 'a a'); + } + + do_execsql_test 3.1.1 { + BEGIN; + DELETE FROM ft ORDER BY rowid LIMIT 3; + SELECT x FROM ft; + ROLLBACK; + } {{a d} {a c} {a b} {a a}} + + do_execsql_test 3.1.2 { + BEGIN; + DELETE FROM ft WHERE ft MATCH 'a' ORDER BY rowid LIMIT 3; + SELECT x FROM ft; + ROLLBACK; + } {{a d} {a c} {a b} {a a}} + + do_execsql_test 3.1.3 { + BEGIN; + DELETE FROM ft WHERE ft MATCH 'b' ORDER BY rowid ASC LIMIT 1 OFFSET 1; + SELECT rowid FROM ft; + ROLLBACK; + } {-45 12 444 12300 25400 50000} + + do_execsql_test 3.2.1 { + BEGIN; + UPDATE ft SET x='hello' ORDER BY rowid LIMIT 2 OFFSET 2; + SELECT x FROM ft; + ROLLBACK; + } {{a a} {a b} hello hello {a c} {a b} {a a}} + + do_execsql_test 3.2.2 { + BEGIN; + UPDATE ft SET x='hello' WHERE ft MATCH 'a' + ORDER BY rowid DESC LIMIT 2 OFFSET 2; + SELECT x FROM ft; + ROLLBACK; + } {{a a} {a b} {a c} hello hello {a b} {a a}} +} ;# fts5 + +#------------------------------------------------------------------------- +# Test using INDEXED BY clauses. +# +foreach {tn t} {1 "" 2 "WITHOUT ROWID"} { + execsql "DROP TABLE IF EXISTS x1" + execsql "CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d) $t" + do_execsql_test 4.$tn.0 { + CREATE INDEX x1bc ON x1(b, c); + INSERT INTO x1 VALUES(1,1,1,1); + INSERT INTO x1 VALUES(2,1,2,2); + INSERT INTO x1 VALUES(3,2,1,3); + INSERT INTO x1 VALUES(4,2,2,3); + INSERT INTO x1 VALUES(5,3,1,2); + INSERT INTO x1 VALUES(6,3,2,1); + } + + do_execsql_test 4.$tn.1 { + BEGIN; + DELETE FROM x1 ORDER BY a LIMIT 2; + SELECT a FROM x1; + ROLLBACK; + } {3 4 5 6} +} + + + + +#------------------------------------------------------------------------- +# Test using object names that require quoting. +# + finish_test From ca3e3c3f1a4c2efdde91a36f35a2ce7df47c02f9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Nov 2017 16:14:26 +0000 Subject: [PATCH 121/156] Fix a problem involving "DELETE/UPDATE...LIMIT" statements that use an INDEXED BY clause. FossilOrigin-Name: 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb --- manifest | 14 +++---- manifest.uuid | 2 +- src/delete.c | 1 + test/wherelimit2.test | 92 ++++++++++++++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 72f23a851b..2d902bb312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\s(DELETE...LIMIT)\sstatements\sagainst\sWITHOUT\sROWID\stables\nwith\sa\ssingle\scolumn\sPK. -D 2017-11-10T15:42:21.918 +C Fix\sa\sproblem\sinvolving\s"DELETE/UPDATE...LIMIT"\sstatements\sthat\suse\san\sINDEXED\nBY\sclause. +D 2017-11-10T16:14:26.225 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 -F src/delete.c cd86beec4f64491270f52521a45f0d335e29766b165511d025ee11119839375a +F src/delete.c 2b9f2d34c9420cbe6d83e99d08d735d8ba6b70729123b8c50909a7fa74a048d3 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 @@ -1558,7 +1558,7 @@ F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8 -F test/wherelimit2.test 565fc74c3f96c3f08f3c20e233530e4c5532e6dfa01d1acc8dd2da5009a3dfd3 +F test/wherelimit2.test 85e3fc3e61e100d8da27323c5cb30ed5881013c3b9100bb26ac5fd52d594c79b F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 @@ -1674,7 +1674,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce -R d15b259ca771d410cbcab13660cb7826 +P 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc +R 07fcc16cd40c06e7e9266df3e5decf1a U dan -Z 4ca2b1e389083d1c64078c78c67450a8 +Z 02c3e7941017d253308184d6859dc937 diff --git a/manifest.uuid b/manifest.uuid index ebde4c85ea..03929fbdcf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc \ No newline at end of file +09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 34c5feb8b7..0d8737756a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -202,6 +202,7 @@ Expr *sqlite3LimitWhere( pSrc->a[0].pTab = 0; pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); pSrc->a[0].pTab = pTab; + pSrc->a[0].pIBIndex = 0; /* generate the SELECT expression tree. */ pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, diff --git a/test/wherelimit2.test b/test/wherelimit2.test index 1bf0168b50..fbcc14aa50 100644 --- a/test/wherelimit2.test +++ b/test/wherelimit2.test @@ -74,7 +74,7 @@ do_execsql_test 1.4 { #------------------------------------------------------------------------- # Simple test using WITHOUT ROWID table. # -do_execsql_test 2.0 { +do_execsql_test 2.1.0 { CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; INSERT INTO t2 VALUES(1, 1, 'h'); INSERT INTO t2 VALUES(1, 2, 'g'); @@ -86,14 +86,14 @@ do_execsql_test 2.0 { INSERT INTO t2 VALUES(4, 2, 'a'); } -do_execsql_test 2.1 { +do_execsql_test 2.1.1 { BEGIN; DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2; SELECT c FROM t2 ORDER BY 1; ROLLBACK; } {a c e f g h} -do_execsql_test 2.2 { +do_execsql_test 2.1.2 { BEGIN; UPDATE t2 SET c=NULL ORDER BY a, b DESC LIMIT 3 OFFSET 1; SELECT a, b, c FROM t2; @@ -109,6 +109,42 @@ do_execsql_test 2.2 { 4 2 a } +do_execsql_test 2.2.0 { + DROP TABLE t2; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 1, 'h'); + INSERT INTO t2 VALUES(2, 2, 'g'); + INSERT INTO t2 VALUES(3, 1, 'f'); + INSERT INTO t2 VALUES(4, 2, 'e'); + INSERT INTO t2 VALUES(5, 1, 'd'); + INSERT INTO t2 VALUES(6, 2, 'c'); + INSERT INTO t2 VALUES(7, 1, 'b'); + INSERT INTO t2 VALUES(8, 2, 'a'); +} + +do_execsql_test 2.2.1 { + BEGIN; + DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2; + SELECT c FROM t2 ORDER BY 1; + ROLLBACK; +} {a c e f g h} + +do_execsql_test 2.2.2 { + BEGIN; + UPDATE t2 SET c=NULL ORDER BY a DESC LIMIT 3 OFFSET 1; + SELECT a, b, c FROM t2; + ROLLBACK; +} { + 1 1 h + 2 2 g + 3 1 f + 4 2 e + 5 1 {} + 6 2 {} + 7 1 {} + 8 2 a +} + #------------------------------------------------------------------------- # Test using a virtual table # @@ -164,29 +200,41 @@ ifcapable fts5 { #------------------------------------------------------------------------- # Test using INDEXED BY clauses. # -foreach {tn t} {1 "" 2 "WITHOUT ROWID"} { - execsql "DROP TABLE IF EXISTS x1" - execsql "CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d) $t" - do_execsql_test 4.$tn.0 { - CREATE INDEX x1bc ON x1(b, c); - INSERT INTO x1 VALUES(1,1,1,1); - INSERT INTO x1 VALUES(2,1,2,2); - INSERT INTO x1 VALUES(3,2,1,3); - INSERT INTO x1 VALUES(4,2,2,3); - INSERT INTO x1 VALUES(5,3,1,2); - INSERT INTO x1 VALUES(6,3,2,1); - } - - do_execsql_test 4.$tn.1 { - BEGIN; - DELETE FROM x1 ORDER BY a LIMIT 2; - SELECT a FROM x1; - ROLLBACK; - } {3 4 5 6} +do_execsql_test 4.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX x1bc ON x1(b, c); + INSERT INTO x1 VALUES(1,1,1,1); + INSERT INTO x1 VALUES(2,1,2,2); + INSERT INTO x1 VALUES(3,2,1,3); + INSERT INTO x1 VALUES(4,2,2,3); + INSERT INTO x1 VALUES(5,3,1,2); + INSERT INTO x1 VALUES(6,3,2,1); } +do_execsql_test 4.1 { + BEGIN; + DELETE FROM x1 ORDER BY a LIMIT 2; + SELECT a FROM x1; + ROLLBACK; +} {3 4 5 6} +do_catchsql_test 4.2 { + DELETE FROM x1 INDEXED BY x1bc WHERE d=3 LIMIT 1; +} {1 {no query solution}} +do_execsql_test 4.3 { + DELETE FROM x1 INDEXED BY x1bc WHERE b=3 LIMIT 1; + SELECT a FROM x1; +} {1 2 3 4 6} + +do_catchsql_test 4.4 { + UPDATE x1 INDEXED BY x1bc SET d=5 WHERE d=3 LIMIT 1; +} {1 {no query solution}} + +do_execsql_test 4.5 { + UPDATE x1 INDEXED BY x1bc SET d=5 WHERE b=2 LIMIT 1; + SELECT a, d FROM x1; +} {1 1 2 2 3 5 4 3 6 1} #------------------------------------------------------------------------- # Test using object names that require quoting. From dc32b448d31e9a22ffa190a60b2561c65d32aebf Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Nov 2017 17:47:54 +0000 Subject: [PATCH 122/156] Add further tests for the code on this branch. FossilOrigin-Name: f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669 --- manifest | 13 +++---- manifest.uuid | 2 +- test/wherelfault.test | 82 +++++++++++++++++++++++++++++++++++++++++++ test/wherelimit2.test | 54 ++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 test/wherelfault.test diff --git a/manifest b/manifest index 2d902bb312..5bd57a17fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sinvolving\s"DELETE/UPDATE...LIMIT"\sstatements\sthat\suse\san\sINDEXED\nBY\sclause. -D 2017-11-10T16:14:26.225 +C Add\sfurther\stests\sfor\sthe\scode\son\sthis\sbranch. +D 2017-11-10T17:47:54.147 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -1557,8 +1557,9 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 +F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8 -F test/wherelimit2.test 85e3fc3e61e100d8da27323c5cb30ed5881013c3b9100bb26ac5fd52d594c79b +F test/wherelimit2.test be78ba3aa1831c6358fd7d5b9809bfd520f0c2a7d63a295e8f182e140ff137c3 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 @@ -1674,7 +1675,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc -R 07fcc16cd40c06e7e9266df3e5decf1a +P 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb +R 276fb9c7ab900388b15c7da93d892ba2 U dan -Z 02c3e7941017d253308184d6859dc937 +Z 620b60e1c2be9813cc742e8964032235 diff --git a/manifest.uuid b/manifest.uuid index 03929fbdcf..1df45c1eee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb \ No newline at end of file +f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669 \ No newline at end of file diff --git a/test/wherelfault.test b/test/wherelfault.test new file mode 100644 index 0000000000..cfb8c1cfb4 --- /dev/null +++ b/test/wherelfault.test @@ -0,0 +1,82 @@ +# 2008 October 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing fault-injection with the +# LIMIT ... OFFSET ... clause of UPDATE and DELETE statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix wherelfault + +ifcapable !update_delete_limit { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 'f'); + INSERT INTO t1 VALUES(2, 'e'); + INSERT INTO t1 VALUES(3, 'd'); + INSERT INTO t1 VALUES(4, 'c'); + INSERT INTO t1 VALUES(5, 'b'); + INSERT INTO t1 VALUES(6, 'a'); + + CREATE VIEW v1 AS SELECT a,b FROM t1; + CREATE TABLE log(op, a); + + CREATE TRIGGER v1del INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete', old.a); + END; + + CREATE TRIGGER v1upd INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update', old.a); + END; +} + +faultsim_save_and_close +do_faultsim_test 1.1 -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { DELETE FROM v1 ORDER BY a LIMIT 3; } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 1.2 -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { UPDATE v1 SET b = 555 ORDER BY a LIMIT 3 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +sqlite3 db test.db +do_execsql_test 2.1.0 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; +} +faultsim_save_and_close + +do_faultsim_test 2.1 -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { DELETE FROM t2 WHERE c=? ORDER BY a DESC LIMIT 10 } +} -test { + faultsim_test_result {0 {}} +} + +finish_test diff --git a/test/wherelimit2.test b/test/wherelimit2.test index fbcc14aa50..43a37958ee 100644 --- a/test/wherelimit2.test +++ b/test/wherelimit2.test @@ -239,7 +239,61 @@ do_execsql_test 4.5 { #------------------------------------------------------------------------- # Test using object names that require quoting. # +do_execsql_test 5.0 { + CREATE TABLE "x y"("a b" PRIMARY KEY, "c d") WITHOUT ROWID; + CREATE INDEX xycd ON "x y"("c d"); + INSERT INTO "x y" VALUES('a', 'a'); + INSERT INTO "x y" VALUES('b', 'b'); + INSERT INTO "x y" VALUES('c', 'c'); + INSERT INTO "x y" VALUES('d', 'd'); + INSERT INTO "x y" VALUES('e', 'a'); + INSERT INTO "x y" VALUES('f', 'b'); + INSERT INTO "x y" VALUES('g', 'c'); + INSERT INTO "x y" VALUES('h', 'd'); +} + +do_execsql_test 5.1 { + BEGIN; + DELETE FROM "x y" WHERE "c d"!='e' ORDER BY "c d" LIMIT 2 OFFSET 2; + SELECT * FROM "x y" ORDER BY 1; + ROLLBACK; +} { + a a c c d d e a g c h d +} + +do_execsql_test 5.2 { + BEGIN; + UPDATE "x y" SET "c d"='e' WHERE "c d"!='e' ORDER BY "c d" LIMIT 2 OFFSET 2; + SELECT * FROM "x y" ORDER BY 1; + ROLLBACK; +} { + a a b e c c d d e a f e g c h d +} + +proc log {args} { lappend ::log {*}$args } +db func log log +do_execsql_test 5.3 { + CREATE VIEW "v w" AS SELECT * FROM "x y"; + CREATE TRIGGER tr1 INSTEAD OF DELETE ON "v w" BEGIN + SELECT log(old."a b", old."c d"); + END; + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON "v w" BEGIN + SELECT log(new."a b", new."c d"); + END; +} + +do_test 5.4 { + set ::log {} + execsql { DELETE FROM "v w" ORDER BY "a b" LIMIT 3 } + set ::log +} {a a b b c c} + +do_test 5.5 { + set ::log {} + execsql { UPDATE "v w" SET "a b" = "a b" || 'x' ORDER BY "a b" LIMIT 5; } + set ::log +} {ax a bx b cx c dx d ex a} finish_test From 85bc6df2f197c3f5f040b8c17c567b1cf5b20e12 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Nov 2017 20:00:50 +0000 Subject: [PATCH 123/156] Improved comments and variable names in the read-only WAL logic. FossilOrigin-Name: d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 --- manifest | 14 +++--- manifest.uuid | 2 +- src/wal.c | 131 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 87 insertions(+), 60 deletions(-) diff --git a/manifest b/manifest index 453bed54e5..c2392bfdac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS. -D 2017-11-09T23:24:29.716 +C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic. +D 2017-11-10T20:00:50.239 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5 +F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed -R 9d6be609689f3743af6adbbc5cf401b2 -U mistachkin -Z 39f42e393c5183ce2049496b416ae681 +P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 +R 4efdec1d3561b29c7744e3cfcf4d5f7b +U drh +Z df395ed2f6beb277e98c44e2de48326a diff --git a/manifest.uuid b/manifest.uuid index ba222182ca..354097e8fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 \ No newline at end of file +d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index afd0e3f7b6..aff0e97fa2 100644 --- a/src/wal.c +++ b/src/wal.c @@ -455,7 +455,7 @@ struct Wal { u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ - u8 bUnlocked; + u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ @@ -1271,7 +1271,7 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); @@ -2099,14 +2099,22 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); - if( rc==SQLITE_READONLY_CANTINIT ){ - assert( page0==0 && pWal->writeLock==0 ); - pWal->bUnlocked = 1; - pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; - *pChanged = 1; - }else if( rc!=SQLITE_OK ){ - return rc; + assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */ + if( rc==SQLITE_READONLY_CANTINIT ){ + /* The SQLITE_READONLY_CANTINIT return means that the shared-memory + ** was openable but is not writable, and this thread is unable to + ** confirm that another write-capable connection has the shared-memory + ** open, and hence the content of the shared-memory is unreliable, + ** since the shared-memory might be inconsistent with the WAL file + ** and there is no writer on hand to fix it. */ + assert( page0==0 && pWal->writeLock==0 ); + pWal->bShmUnreliable = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else{ + return rc; /* Any other non-OK return is just an error */ + } }; assert( page0 || pWal->writeLock==0 ); @@ -2122,7 +2130,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; @@ -2152,10 +2160,10 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } - if( pWal->bUnlocked ){ + if( pWal->bShmUnreliable ){ if( rc!=SQLITE_OK ){ walIndexClose(pWal, 0); - pWal->bUnlocked = 0; + pWal->bShmUnreliable = 0; assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } @@ -2166,11 +2174,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ } /* -** Open an "unlocked" transaction. An unlocked transaction is a read -** transaction used by a read-only client in cases where the *-shm -** file cannot be mapped and its contents cannot be trusted. It is -** assumed that the *-wal file has been read and that a wal-index -** constructed in heap memory is currently available in Wal.apWiData[]. +** Open a transaction in a connection where the shared-memory is read-only +** and where we cannot verify that there is a separate write-capable connection +** on hand to keep the shared-memory up-to-date with the WAL file. +** +** This can happen, for example, when the shared-memory is implemented by +** memory-mapping a *-shm file, where a prior writer has shut down and +** left the *-shm file on disk, and now the present connection is trying +** to use that database but lacks write permission on the *-shm file. +** Other scenarios are also possible, depending on the VFS implementation. +** +** Precondition: +** +** The *-wal file has been read and an appropriate wal-index has been +** constructed in pWal->apWiData[] using heap memory instead of shared +** memory. ** ** If this function returns SQLITE_OK, then the read transaction has ** been successfully opened. In this case output variable (*pChanged) @@ -2182,7 +2200,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** ** If an error occurs, an SQLite error code is returned. */ -static int walBeginUnlocked(Wal *pWal, int *pChanged){ +static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ i64 szWal; /* Size of wal file on disk in bytes */ i64 iOffset; /* Current offset when reading wal file */ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ @@ -2193,50 +2211,59 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ int rc; /* Return code */ u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ - assert( pWal->bUnlocked ); + assert( pWal->bShmUnreliable ); assert( pWal->readOnly & WAL_SHM_RDONLY ); assert( pWal->nWiData>0 && pWal->apWiData[0] ); /* Take WAL_READ_LOCK(0). This has the effect of preventing any - ** live clients from running a checkpoint, but does not stop them + ** writers from running a checkpoint, but does not stop them ** from running recovery. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); if( rc!=SQLITE_OK ){ if( rc==SQLITE_BUSY ) rc = WAL_RETRY; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } pWal->readLock = 0; - /* Try to map the *-shm file again. If it succeeds this time, then - ** a non-readonly_shm connection has already connected to the database. - ** In this case, start over with opening the transaction. + /* Check to see if a separate writer has attached to the shared-memory area, + ** thus making the shared-memory "reliable" again. Do this by invoking + ** the xShmMap() routine of the VFS and looking to see if the return + ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. ** - ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never - ** return SQLITE_OK here, as that would imply that it had established - ** a read/write mapping. A return of SQLITE_READONLY means success - that - ** a mapping has been established to a shared-memory segment that is actively - ** maintained by a writer. SQLITE_READONLY_CANTINIT means that all - ** all connections to the -shm file are read-only and hence the content - ** of the -shm file might be out-of-date. - ** - ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint - ** from taking place. But it does not prevent the wal from being wrapped - ** if a checkpoint has already taken place. This means that if another - ** client is connected at this point, it may have already checkpointed - ** the entire wal. In that case it would not be safe to continue with - ** the unlocked transaction, as the other client may overwrite wal - ** frames that this client is still using. */ + ** Once sqlite3OsShmMap() has been called for a file and has returned + ** any SQLITE_READONLY value, it must SQLITE_READONLY or + ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** until sqlite3OsShmUnmap() has been called. This is a requirement + ** on the VFS implementation. + ** + ** If the shared-memory is now "reliable" return WAL_RETRY, which will + ** cause the heap-memory WAL-index to be discarded and the actual + ** shared memory to be used in its place. + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } + /* Reach this point only if the real shared-memory is still unreliable. + ** Assume the in-memory WAL-index substitute is correct and load it + ** into pWal->hdr. + */ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint + ** from taking place. But it does not prevent the wal from being wrapped + ** if a checkpoint has already taken place. This means that if another + ** client is connected at this point, it may have already checkpointed + ** the entire wal. In that case it would not be safe to continue with + ** the this transaction, as the other client may overwrite wal + ** frames that this client is still using. + */ rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); if( rc!=SQLITE_OK ){ - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } /* Check the salt keys at the start of the wal file still match. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); if( rc!=SQLITE_OK ){ - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ rc = WAL_RETRY; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } /* Allocate a buffer to read frames into */ @@ -2265,7 +2292,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ aFrame = (u8 *)sqlite3_malloc64(szFrame); if( aFrame==0 ){ rc = SQLITE_NOMEM_BKPT; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -2298,7 +2325,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; - begin_unlocked_out: + begin_unreliable_shm_out: sqlite3_free(aFrame); if( rc!=SQLITE_OK ){ int i; @@ -2306,7 +2333,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ sqlite3_free((void*)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - pWal->bUnlocked = 0; + pWal->bShmUnreliable = 0; sqlite3WalEndReadTransaction(pWal); *pChanged = 1; } @@ -2402,7 +2429,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( !useWal ){ assert( rc==SQLITE_OK ); - if( pWal->bUnlocked==0 ){ + if( pWal->bShmUnreliable==0 ){ rc = walIndexReadHdr(pWal, pChanged); } if( rc==SQLITE_BUSY ){ @@ -2433,8 +2460,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( rc!=SQLITE_OK ){ return rc; } - else if( pWal->bUnlocked ){ - return walBeginUnlocked(pWal, pChanged); + else if( pWal->bShmUnreliable ){ + return walBeginShmUnreliable(pWal, pChanged); } } @@ -2789,7 +2816,7 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } @@ -2852,7 +2879,7 @@ int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->bUnlocked || pWal->minFrame>0 ); + assert( pWal->bShmUnreliable || pWal->minFrame>0 ); for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; From 3b61ebb8de161ffdc505d83d9acd9b881bb139bf Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Nov 2017 20:13:14 +0000 Subject: [PATCH 124/156] Omit some extra code from non-SQLITE_ENABLE_UPDATE_DELETE_LIMIT builds. FossilOrigin-Name: 72be33f9c84de3ec4afc40549482417456ca82c1d16b473dc034b144055271e5 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/delete.c | 10 ++++++---- src/fkey.c | 2 +- src/parse.y | 8 ++++---- src/resolve.c | 5 +---- src/sqliteInt.h | 7 ++----- src/trigger.c | 4 ++-- src/update.c | 10 ++++++---- 9 files changed, 35 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 5bd57a17fd..e5b0bb5863 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sthe\scode\son\sthis\sbranch. -D 2017-11-10T17:47:54.147 +C Omit\ssome\sextra\scode\sfrom\snon-SQLITE_ENABLE_UPDATE_DELETE_LIMIT\sbuilds. +D 2017-11-10T20:13:14.097 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -422,10 +422,10 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 -F src/delete.c 2b9f2d34c9420cbe6d83e99d08d735d8ba6b70729123b8c50909a7fa74a048d3 +F src/delete.c f97cc2a9a30b4119ec5ed0da4e41335060cbb8510a5f1a756be48c065cb824f0 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 +F src/fkey.c 821f70b6c43ba4542ffb5fdf11d3ac5f79e4cbae4750ee3de379afbc5e1f7781 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a @@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 49eb13a590f88a03e81f1bc8df0634e56598ebd8d76d38dda24ba9edd7f92194 +F src/parse.y f5f02ef39444982af36545bd52ae2921fc4ba1bac24d9b11efd8ec1f52c5b4dc F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -468,14 +468,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 704978e5aabb9f524789e3b53016b4068a4e64c669c0f8ff025d2b23b95d62b5 +F src/resolve.c 5b1e89ba279f4a4ab2f0975a7100d75be71e1a43a2df75a9c909d45bdd18c6ed F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h 821479be791acfdb63177b21a491e604526191c1630d1d65d7a4a7c65d203c83 +F src/sqliteInt.h abd4e64bc72906449d801d0e211265525239bc021bd9b7a72143c281fc24fa03 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -534,8 +534,8 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef -F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc -F src/update.c 5123fcb4aa98a705ced2acb172c1761a570d142d77901b4f4ce38acb191ef8ed +F src/trigger.c 929b12bb5f9ab3b10c589f71fa176effe6ea8bddc9eb1bbd2439755fa0acdfdf +F src/update.c 2e7ef3839eb620e9140a41bbfd05e29d7d216441c5d97221b75635dea4e49672 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 @@ -1675,7 +1675,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb -R 276fb9c7ab900388b15c7da93d892ba2 +P f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669 +R 1176f1b5b2e02ce4a3ab41b0af2d65f4 U dan -Z 620b60e1c2be9813cc742e8964032235 +Z f8079371239b2151fe43a05b53b9cf38 diff --git a/manifest.uuid b/manifest.uuid index 1df45c1eee..4ef7d391b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669 \ No newline at end of file +72be33f9c84de3ec4afc40549482417456ca82c1d16b473dc034b144055271e5 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 0d8737756a..5c96fdfb8d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -224,13 +224,13 @@ Expr *sqlite3LimitWhere( ** \________/ \________________/ ** pTabList pWhere */ -void sqlite3DeleteFromLimit( +void sqlite3DeleteFrom( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table from which we should delete things */ Expr *pWhere, /* The WHERE clause. May be null */ - ExprList *pOrderBy, - Expr *pLimit, - Expr *pOffset + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit, /* LIMIT clause. May be null */ + Expr *pOffset /* OFFSET clause. May be null */ ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ @@ -606,9 +606,11 @@ delete_from_cleanup: sqlite3AuthContextPop(&sContext); sqlite3SrcListDelete(db, pTabList); sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); sqlite3ExprDelete(db, pOffset); +#endif sqlite3DbFree(db, aToOpen); return; } diff --git a/src/fkey.c b/src/fkey.c index 0012768a39..44ef3c7c7b 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -725,7 +725,7 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ } pParse->disableTriggers = 1; - sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0); + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0, 0); pParse->disableTriggers = 0; /* If the DELETE has generated immediate foreign key constraint diff --git a/src/parse.y b/src/parse.y index 2a536b0a8e..18f587c05d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -753,14 +753,14 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3DeleteFromLimit(pParse,X,W,O,L.pLimit,L.pOffset); + sqlite3DeleteFrom(pParse,X,W,O,L.pLimit,L.pOffset); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3DeleteFrom(pParse,X,W); + sqlite3DeleteFrom(pParse,X,W,0,0,0); } %endif @@ -778,7 +778,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); - sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset); + sqlite3Update(pParse,X,Y,W,R,O,L.pLimit,L.pOffset); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT @@ -787,7 +787,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); - sqlite3Update(pParse,X,Y,W,R); + sqlite3Update(pParse,X,Y,W,R,0,0,0); } %endif diff --git a/src/resolve.c b/src/resolve.c index 945654ead0..17dbbccfc5 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -597,10 +597,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ struct SrcList_item *pItem; assert( pSrcList && pSrcList->nSrc==1 ); pItem = pSrcList->a; - if( !HasRowid(pItem->pTab) || pItem->pTab->pSelect!=0 ){ - sqlite3ErrorMsg(pParse, "ORDER BY and LIMIT not support for table %s", - pItem->pTab->zName); - } + assert( HasRowid(pItem->pTab) && pItem->pTab->pSelect==0 ); pExpr->op = TK_COLUMN; pExpr->pTab = pItem->pTab; pExpr->iTable = pItem->iCursor; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d3c412e222..95a66b4fe7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3764,11 +3764,8 @@ void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif -void sqlite3DeleteFromLimit(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*); -#define sqlite3DeleteFrom(x,y,z) sqlite3DeleteFromLimit(x,y,z,0,0,0) -void sqlite3UpdateLimit(Parse*, SrcList*, ExprList*, Expr*, int, - ExprList*,Expr*,Expr*); -#define sqlite3Update(v,w,x,y,z) sqlite3UpdateLimit(v,w,x,y,z,0,0,0) +void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*); +void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,Expr*); WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); LogEst sqlite3WhereOutputRowCount(WhereInfo*); diff --git a/src/trigger.c b/src/trigger.c index 2a4fdd8ed4..a17769ae98 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -711,7 +711,7 @@ static int codeTriggerProgram( targetSrcList(pParse, pStep), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), - pParse->eOrconf + pParse->eOrconf, 0, 0, 0 ); break; } @@ -727,7 +727,7 @@ static int codeTriggerProgram( case TK_DELETE: { sqlite3DeleteFrom(pParse, targetSrcList(pParse, pStep), - sqlite3ExprDup(db, pStep->pWhere, 0) + sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0, 0 ); break; } diff --git a/src/update.c b/src/update.c index f63a9ffe8a..4cfa4019eb 100644 --- a/src/update.c +++ b/src/update.c @@ -86,15 +86,15 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ ** \_______/ \________/ \______/ \________________/ * onError pTabList pChanges pWhere */ -void sqlite3UpdateLimit( +void sqlite3Update( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table in which we should change things */ ExprList *pChanges, /* Things to be changed */ Expr *pWhere, /* The WHERE clause. May be null */ int onError, /* How to handle constraint errors */ - ExprList *pOrderBy, - Expr *pLimit, - Expr *pOffset + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit, /* LIMIT clause. May be null */ + Expr *pOffset /* OFFSET clause. May be null */ ){ int i, j; /* Loop counters */ Table *pTab; /* The table to be updated */ @@ -745,9 +745,11 @@ update_cleanup: sqlite3SrcListDelete(db, pTabList); sqlite3ExprListDelete(db, pChanges); sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); sqlite3ExprDelete(db, pOffset); +#endif return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise From 870655bb9e7ef261995c2f3dc6fa5f6c9b115997 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Nov 2017 13:30:44 +0000 Subject: [PATCH 125/156] Further comment improvements in wal.c. No code changes. FossilOrigin-Name: 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 35 ++++++++++++++++++++--------------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index c2392bfdac..e067bdd554 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic. -D 2017-11-10T20:00:50.239 +C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges. +D 2017-11-11T13:30:44.595 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420 +F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 -R 4efdec1d3561b29c7744e3cfcf4d5f7b +P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 +R cc050d490699d35569ec56a8515bac08 U drh -Z df395ed2f6beb277e98c44e2de48326a +Z 1e870a391397f9beedb8e100937bac44 diff --git a/manifest.uuid b/manifest.uuid index 354097e8fa..3a4ec03265 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 \ No newline at end of file +346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index aff0e97fa2..87fbda12d6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2230,16 +2230,23 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** the xShmMap() routine of the VFS and looking to see if the return ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. ** - ** Once sqlite3OsShmMap() has been called for a file and has returned - ** any SQLITE_READONLY value, it must SQLITE_READONLY or - ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, - ** until sqlite3OsShmUnmap() has been called. This is a requirement - ** on the VFS implementation. - ** ** If the shared-memory is now "reliable" return WAL_RETRY, which will ** cause the heap-memory WAL-index to be discarded and the actual ** shared memory to be used in its place. - */ + ** + ** This step is important because, even though this connection is holding + ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might + ** have already checkpointed the WAL file and, while the current + ** is active, wrap the WAL and start overwriting frames that this + ** process wants to use. + ** + ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has + ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY + ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** even if some external agent does a "chmod" to make the shared-memory + ** writable by us, until sqlite3OsShmUnmap() has been called. + ** This is a requirement on the VFS implementation. + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ @@ -2247,19 +2254,14 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ goto begin_unreliable_shm_out; } - /* Reach this point only if the real shared-memory is still unreliable. + /* We reach this point only if the real shared-memory is still unreliable. ** Assume the in-memory WAL-index substitute is correct and load it ** into pWal->hdr. */ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); - /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint - ** from taking place. But it does not prevent the wal from being wrapped - ** if a checkpoint has already taken place. This means that if another - ** client is connected at this point, it may have already checkpointed - ** the entire wal. In that case it would not be safe to continue with - ** the this transaction, as the other client may overwrite wal - ** frames that this client is still using. + /* Make sure some writer hasn't come in and changed the WAL file out + ** from under us, then disconnected, while we were not looking. */ rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); if( rc!=SQLITE_OK ){ @@ -2283,6 +2285,9 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ goto begin_unreliable_shm_out; } if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + /* Some writer has wrapped the WAL file while we were not looking. + ** Return WAL_RETRY which will cause the in-memory WAL-index to be + ** rebuilt. */ rc = WAL_RETRY; goto begin_unreliable_shm_out; } From c05a063c680af17386b221b977bc4eda1cf56e2b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Nov 2017 20:11:01 +0000 Subject: [PATCH 126/156] In wal.c: improved comments, new assert() and testcase() macros, and replace some magic numbers with appropriate symbolic constants. FossilOrigin-Name: 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 33 +++++++++++++++++++++++---------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index e067bdd554..2fbd04081b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges. -D 2017-11-11T13:30:44.595 +C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants. +D 2017-11-11T20:11:01.646 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07 +F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 -R cc050d490699d35569ec56a8515bac08 +P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e +R cc8d47b78b14e89aa0bda73b44dcaa29 U drh -Z 1e870a391397f9beedb8e100937bac44 +Z f012da86d6540243e580ab3866be51e7 diff --git a/manifest.uuid b/manifest.uuid index 3a4ec03265..cbdfc58fda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e \ No newline at end of file +13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 87fbda12d6..6fddf39076 100644 --- a/src/wal.c +++ b/src/wal.c @@ -545,6 +545,11 @@ struct WalIterator { ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are ** numbered from zero. ** +** If the wal-index is currently smaller the iPage pages then the size +** of the wal-index might be increased, but only if it is safe to do +** so. It is safe to enlarge the wal-index if pWal->writeLock is true +** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE. +** ** If this call is successful, *ppPage is set to point to the wal-index ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, ** then an SQLite error code is returned and *ppPage is set to 0. @@ -576,6 +581,8 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; if( rc==SQLITE_READONLY ){ @@ -2108,15 +2115,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** open, and hence the content of the shared-memory is unreliable, ** since the shared-memory might be inconsistent with the WAL file ** and there is no writer on hand to fix it. */ - assert( page0==0 && pWal->writeLock==0 ); + assert( page0==0 ); + assert( pWal->writeLock==0 ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); pWal->bShmUnreliable = 1; pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; *pChanged = 1; }else{ return rc; /* Any other non-OK return is just an error */ } - }; - assert( page0 || pWal->writeLock==0 ); + }else{ + /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock + ** is zero, which prevents the SHM from growing */ + testcase( page0!=0 ); + } + assert( page0!=0 || pWal->writeLock==0 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually @@ -3661,24 +3674,24 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ - if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; + if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_NORMAL_MODE; if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; }else{ /* Already in locking_mode=NORMAL */ rc = 0; } }else if( op>0 ){ - assert( pWal->exclusiveMode==0 ); + assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; } return rc; } From 2e9b0923aca34aa5d18a3be65f7e082c45758cc3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Nov 2017 05:51:37 +0000 Subject: [PATCH 127/156] Remove some branches in walTryBeginRead() that were added by check-in [ce5d13c2de] but became unreachable with the addition of logic in check-in [18b26843] that enabled read-only clients to parse the WAL file into a heap-memory WAL-index, thus guaranteeing that the WAL-index header is always available. FossilOrigin-Name: 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 +++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2fbd04081b..fc71ea63a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants. -D 2017-11-11T20:11:01.646 +C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable. +D 2017-11-13T05:51:37.627 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c +F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e -R cc8d47b78b14e89aa0bda73b44dcaa29 +P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b +R be755a48b3ce9c3ee36556946fccdf2c U drh -Z f012da86d6540243e580ab3866be51e7 +Z a296aa345abf23e8a36cfa010f746e55 diff --git a/manifest.uuid b/manifest.uuid index cbdfc58fda..0dbf02629b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b \ No newline at end of file +9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 6fddf39076..382be4e0c8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2418,6 +2418,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->readLock<0 ); /* Not currently locked */ + /* useWal may only be set for read/write connections */ + assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 ); + /* Take steps to avoid spinning forever if there is a protocol error. ** ** Circumstances that cause a RETRY should only last for the briefest @@ -2484,9 +2487,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } assert( pWal->nWiData>0 ); - assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) ); - pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0; - if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame) + assert( pWal->apWiData[0]!=0 ); + pInfo = walCkptInfo(pWal); + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) @@ -2498,9 +2501,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(0)); walShmBarrier(pWal); if( rc==SQLITE_OK ){ - if( pInfo - && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, From 8b17ac1919afa4f2c73869e30f6938db775ea779 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Nov 2017 03:42:52 +0000 Subject: [PATCH 128/156] Improvement to a comment. No changes to code. FossilOrigin-Name: 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fc71ea63a6..50ac558bb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable. -D 2017-11-13T05:51:37.627 +C Improvement\sto\sa\scomment.\s\sNo\schanges\sto\scode. +D 2017-11-14T03:42:52.056 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510 +F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b -R be755a48b3ce9c3ee36556946fccdf2c +P 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 +R 696f834341714d48762ee99cfc8345fc U drh -Z a296aa345abf23e8a36cfa010f746e55 +Z 27bc2b0f1009e1afc7034f11c759006b diff --git a/manifest.uuid b/manifest.uuid index 0dbf02629b..b0b16a61f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 \ No newline at end of file +486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 382be4e0c8..376151e1a8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2178,6 +2178,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walIndexClose(pWal, 0); pWal->bShmUnreliable = 0; assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + /* walIndexRecover() might have returned SHORT_READ if a concurrent + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } pWal->exclusiveMode = WAL_NORMAL_MODE; From 8c0833fb218ef0b3fb89bef2b9600834e74cf8b7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Nov 2017 23:48:23 +0000 Subject: [PATCH 129/156] In the parse tree, combine LIMIT and OFFSET into a single expression rooted on a TK_LIMIT node, for a small code size reduction and performance increase, and a reduction in code complexity. FossilOrigin-Name: 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826 --- manifest | 37 +++++++++++++++--------------- manifest.uuid | 2 +- src/attach.c | 3 --- src/delete.c | 22 ++++++------------ src/expr.c | 14 +++++++----- src/fkey.c | 4 ++-- src/insert.c | 1 - src/parse.y | 45 ++++++++++++++----------------------- src/resolve.c | 3 +-- src/select.c | 60 +++++++++++++++++-------------------------------- src/sqliteInt.h | 11 +++++---- src/treeview.c | 13 +++++------ src/trigger.c | 4 ++-- src/update.c | 12 +++++----- src/walker.c | 1 - 15 files changed, 92 insertions(+), 140 deletions(-) diff --git a/manifest b/manifest index f1fb27c3db..1475fea8d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\sread\sfrom\sread-only\sWAL-mode\sdatabase\sfiles\sas\slong\sas\nthe\s-wal\sand\s-shm\sfiles\sare\spresent\son\sdisk. -D 2017-11-14T19:34:22.764 +C In\sthe\sparse\stree,\scombine\sLIMIT\sand\sOFFSET\sinto\sa\ssingle\sexpression\srooted\non\sa\sTK_LIMIT\snode,\sfor\sa\ssmall\scode\ssize\sreduction\sand\sperformance\sincrease,\nand\sa\sreduction\sin\scode\scomplexity. +D 2017-11-14T23:48:23.099 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -407,7 +407,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594 F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 -F src/attach.c 07b706e336fd3cedbd855e1f8266d10e82fecae07daf86717b5760cd7784c584 +F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 @@ -422,17 +422,17 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 -F src/delete.c f97cc2a9a30b4119ec5ed0da4e41335060cbb8510a5f1a756be48c065cb824f0 -F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 +F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3 +F src/expr.c 5257a9157f22f048ddcce5cd494d39633e89c2a4769671311b3e7875d262f746 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 821f70b6c43ba4542ffb5fdf11d3ac5f79e4cbae4750ee3de379afbc5e1f7781 +F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2 +F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5 @@ -459,7 +459,7 @@ F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y f5f02ef39444982af36545bd52ae2921fc4ba1bac24d9b11efd8ec1f52c5b4dc +F src/parse.y d79001da275bfe344b409006b85e81e486a0f6afc3762fdf0944f000f4aa0111 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -468,14 +468,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 5b1e89ba279f4a4ab2f0975a7100d75be71e1a43a2df75a9c909d45bdd18c6ed +F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee +F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h abd4e64bc72906449d801d0e211265525239bc021bd9b7a72143c281fc24fa03 +F src/sqliteInt.h fb297e4b891608057e857d583e30a261d905a3f41493f351fc91bae7d22008ff F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -533,9 +533,9 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 -F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef -F src/trigger.c 929b12bb5f9ab3b10c589f71fa176effe6ea8bddc9eb1bbd2439755fa0acdfdf -F src/update.c 2e7ef3839eb620e9140a41bbfd05e29d7d216441c5d97221b75635dea4e49672 +F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd +F src/trigger.c fc6be2a6e103d9e38b161e07d7db0ffb1f2218bd2f27ccdc0a3d1cc89e9cea0f +F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 @@ -552,7 +552,7 @@ F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a -F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f +F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 @@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 -R 4dd08ae7170c8d270fe307a98bb5bfb2 -T +closed 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 +P 00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3 +R 9fe4e3d4453a36374c8a196662e2c5b0 U drh -Z 1fe131254421255cb83ffd7abf325441 +Z 3ab594fe26a5b432ce6d9e25b1b1040f diff --git a/manifest.uuid b/manifest.uuid index 1aae26850e..41cb7f1619 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3 \ No newline at end of file +3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 68a9a77c66..fa38e84159 100644 --- a/src/attach.c +++ b/src/attach.c @@ -504,9 +504,6 @@ int sqlite3FixSelect( if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ return 1; } - if( sqlite3FixExpr(pFix, pSelect->pOffset) ){ - return 1; - } pSelect = pSelect->pPrior; } return 0; diff --git a/src/delete.c b/src/delete.c index 5c96fdfb8d..8e25b36f5a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -92,7 +92,6 @@ void sqlite3MaterializeView( Expr *pWhere, /* Optional WHERE clause to be added */ ExprList *pOrderBy, /* Optional ORDER BY clause */ Expr *pLimit, /* Optional LIMIT clause */ - Expr *pOffset, /* Optional OFFSET clause */ int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; @@ -110,7 +109,7 @@ void sqlite3MaterializeView( assert( pFrom->a[0].pUsing==0 ); } pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, - SF_IncludeHidden, pLimit, pOffset); + SF_IncludeHidden, pLimit); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -132,7 +131,6 @@ Expr *sqlite3LimitWhere( Expr *pWhere, /* The WHERE clause. May be null */ ExprList *pOrderBy, /* The ORDER BY clause. May be null */ Expr *pLimit, /* The LIMIT clause. May be null */ - Expr *pOffset, /* The OFFSET clause. May be null */ char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ ){ sqlite3 *db = pParse->db; @@ -149,8 +147,6 @@ Expr *sqlite3LimitWhere( sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); sqlite3ExprDelete(pParse->db, pWhere); sqlite3ExprListDelete(pParse->db, pOrderBy); - sqlite3ExprDelete(pParse->db, pLimit); - sqlite3ExprDelete(pParse->db, pOffset); return 0; } @@ -158,8 +154,6 @@ Expr *sqlite3LimitWhere( ** is a limit/offset term to enforce. */ if( pLimit == 0 ) { - /* if pLimit is null, pOffset will always be null as well. */ - assert( pOffset == 0 ); return pWhere; } @@ -206,7 +200,7 @@ Expr *sqlite3LimitWhere( /* generate the SELECT expression tree. */ pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, - pOrderBy,0,pLimit,pOffset + pOrderBy,0,pLimit ); /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ @@ -229,8 +223,7 @@ void sqlite3DeleteFrom( SrcList *pTabList, /* The table from which we should delete things */ Expr *pWhere, /* The WHERE clause. May be null */ ExprList *pOrderBy, /* ORDER BY clause. May be null */ - Expr *pLimit, /* LIMIT clause. May be null */ - Expr *pOffset /* OFFSET clause. May be null */ + Expr *pLimit /* LIMIT clause. May be null */ ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ @@ -303,10 +296,10 @@ void sqlite3DeleteFrom( #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT if( !isView ){ pWhere = sqlite3LimitWhere( - pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "DELETE" + pParse, pTabList, pWhere, pOrderBy, pLimit, "DELETE" ); pOrderBy = 0; - pLimit = pOffset = 0; + pLimit = 0; } #endif @@ -358,11 +351,11 @@ void sqlite3DeleteFrom( #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ sqlite3MaterializeView(pParse, pTab, - pWhere, pOrderBy, pLimit, pOffset, iTabCur + pWhere, pOrderBy, pLimit, iTabCur ); iDataCur = iIdxCur = iTabCur; pOrderBy = 0; - pLimit = pOffset = 0; + pLimit = 0; } #endif @@ -609,7 +602,6 @@ delete_from_cleanup: #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); - sqlite3ExprDelete(db, pOffset); #endif sqlite3DbFree(db, aToOpen); return; diff --git a/src/expr.c b/src/expr.c index 0a5dc913f6..5027f994db 100644 --- a/src/expr.c +++ b/src/expr.c @@ -663,7 +663,6 @@ static void heightOfSelect(Select *p, int *pnHeight){ heightOfExpr(p->pWhere, pnHeight); heightOfExpr(p->pHaving, pnHeight); heightOfExpr(p->pLimit, pnHeight); - heightOfExpr(p->pOffset, pnHeight); heightOfExprList(p->pEList, pnHeight); heightOfExprList(p->pGroupBy, pnHeight); heightOfExprList(p->pOrderBy, pnHeight); @@ -1462,7 +1461,6 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ pNew->pNext = pNext; pNew->pPrior = 0; pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); pNew->iLimit = 0; pNew->iOffset = 0; pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; @@ -2099,7 +2097,6 @@ static Select *isCandidateForInOpt(Expr *pX){ } assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ if( p->pLimit ) return 0; /* Has no LIMIT clause */ - assert( p->pOffset==0 ); /* No LIMIT means no OFFSET */ if( p->pWhere ) return 0; /* Has no WHERE clause */ pSrc = p->pSrc; assert( pSrc!=0 ); @@ -2739,6 +2736,7 @@ int sqlite3CodeSubselect( Select *pSel; /* SELECT statement to encode */ SelectDest dest; /* How to deal with SELECT result */ int nReg; /* Registers to allocate */ + Expr *pLimit; /* New limit expression */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); @@ -2760,9 +2758,13 @@ int sqlite3CodeSubselect( sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); VdbeComment((v, "Init EXISTS result")); } - sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER, - &sqlite3IntTokens[1], 0); + pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[1], 0); + if( pSel->pLimit ){ + sqlite3ExprDelete(pParse->db, pSel->pLimit->pLeft); + pSel->pLimit->pLeft = pLimit; + }else{ + pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); + } pSel->iLimit = 0; pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ diff --git a/src/fkey.c b/src/fkey.c index 44ef3c7c7b..c366c1b3aa 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -725,7 +725,7 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ } pParse->disableTriggers = 1; - sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0, 0); + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0); pParse->disableTriggers = 0; /* If the DELETE has generated immediate foreign key constraint @@ -1283,7 +1283,7 @@ static Trigger *fkActionTrigger( sqlite3ExprListAppend(pParse, 0, pRaise), sqlite3SrcListAppend(db, 0, &tFrom, 0), pWhere, - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0 ); pWhere = 0; } diff --git a/src/insert.c b/src/insert.c index 3a531495cf..f0af0fbd10 100644 --- a/src/insert.c +++ b/src/insert.c @@ -2007,7 +2007,6 @@ static int xferOptimization( if( pSelect->pLimit ){ return 0; /* SELECT may not have a LIMIT clause */ } - assert( pSelect->pOffset==0 ); /* Must be so if pLimit==0 */ if( pSelect->pPrior ){ return 0; /* SELECT may not be a compound query */ } diff --git a/src/parse.y b/src/parse.y index 18f587c05d..0867d1e0c8 100644 --- a/src/parse.y +++ b/src/parse.y @@ -84,15 +84,6 @@ */ #define YYMALLOCARGTYPE u64 -/* -** An instance of this structure holds information about the -** LIMIT clause of a SELECT statement. -*/ -struct LimitVal { - Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ - Expr *pOffset; /* The OFFSET expression. NULL if there is none */ -}; - /* ** An instance of the following structure describes the event of a ** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, @@ -470,7 +461,7 @@ selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z). { x.n = 0; parserDoubleLinkSelect(pParse, pRhs); pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); - pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ pRhs->op = (u8)Y; @@ -493,7 +484,7 @@ oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) #if SELECTTRACE_ENABLED Token s = S; /*A-overwrites-S*/ #endif - A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); + A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L); #if SELECTTRACE_ENABLED /* Populate the Select.zSelName[] string that is used to help with ** query planner debugging, to differentiate between multiple Select @@ -524,11 +515,11 @@ oneselect(A) ::= values(A). %type values {Select*} %destructor values {sqlite3SelectDelete(pParse->db, $$);} values(A) ::= VALUES LP nexprlist(X) RP. { - A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0); + A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0); } values(A) ::= values(A) COMMA LP exprlist(Y) RP. { Select *pRight, *pLeft = A; - pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); + pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0); if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; @@ -639,7 +630,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) }else{ Select *pSubquery; sqlite3SrcListShiftJoinType(F); - pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0,0); + pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0); A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,pSubquery,N,U); } } @@ -726,7 +717,7 @@ groupby_opt(A) ::= GROUP BY nexprlist(X). {A = X;} having_opt(A) ::= . {A = 0;} having_opt(A) ::= HAVING expr(X). {A = X.pExpr;} -%type limit_opt {struct LimitVal} +%type limit_opt {Expr*} // The destructor for limit_opt will never fire in the current grammar. // The limit_opt non-terminal only occurs at the end of a single production @@ -735,16 +726,14 @@ having_opt(A) ::= HAVING expr(X). {A = X.pExpr;} // reduce. So there is never a limit_opt non-terminal on the stack // except as a transient. So there is never anything to destroy. // -//%destructor limit_opt { -// sqlite3ExprDelete(pParse->db, $$.pLimit); -// sqlite3ExprDelete(pParse->db, $$.pOffset); -//} -limit_opt(A) ::= . {A.pLimit = 0; A.pOffset = 0;} -limit_opt(A) ::= LIMIT expr(X). {A.pLimit = X.pExpr; A.pOffset = 0;} +//%destructor limit_opt {sqlite3ExprDelete(pParse->db, $$);} +limit_opt(A) ::= . {A = 0;} +limit_opt(A) ::= LIMIT expr(X). + {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,0);} limit_opt(A) ::= LIMIT expr(X) OFFSET expr(Y). - {A.pLimit = X.pExpr; A.pOffset = Y.pExpr;} + {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,Y.pExpr);} limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). - {A.pOffset = X.pExpr; A.pLimit = Y.pExpr;} + {A = sqlite3PExpr(pParse,TK_LIMIT,Y.pExpr,X.pExpr);} /////////////////////////// The DELETE statement ///////////////////////////// // @@ -753,14 +742,14 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3DeleteFrom(pParse,X,W,O,L.pLimit,L.pOffset); + sqlite3DeleteFrom(pParse,X,W,O,L); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3DeleteFrom(pParse,X,W,0,0,0); + sqlite3DeleteFrom(pParse,X,W,0,0); } %endif @@ -778,7 +767,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); - sqlite3Update(pParse,X,Y,W,R,O,L.pLimit,L.pOffset); + sqlite3Update(pParse,X,Y,W,R,O,L); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT @@ -787,7 +776,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); - sqlite3Update(pParse,X,Y,W,R,0,0,0); + sqlite3Update(pParse,X,Y,W,R,0,0); } %endif @@ -1189,7 +1178,7 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { } expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z); - Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); if( E ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E); A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0); sqlite3PExprAddSelect(pParse, A.pExpr, pSelect); diff --git a/src/resolve.c b/src/resolve.c index 17dbbccfc5..f735fffa02 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1196,8 +1196,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; - if( sqlite3ResolveExprNames(&sNC, p->pLimit) || - sqlite3ResolveExprNames(&sNC, p->pOffset) ){ + if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ return WRC_Abort; } diff --git a/src/select.c b/src/select.c index 46c69d71fb..97eaf21b4d 100644 --- a/src/select.c +++ b/src/select.c @@ -74,7 +74,6 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); - sqlite3ExprDelete(db, p->pOffset); if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); p = pPrior; @@ -107,8 +106,7 @@ Select *sqlite3SelectNew( Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ u32 selFlags, /* Flag parameters, such as SF_Distinct */ - Expr *pLimit, /* LIMIT value. NULL means not used */ - Expr *pOffset /* OFFSET value. NULL means no offset */ + Expr *pLimit /* LIMIT value. NULL means not used */ ){ Select *pNew; Select standin; @@ -141,10 +139,7 @@ Select *sqlite3SelectNew( pNew->pPrior = 0; pNew->pNext = 0; pNew->pLimit = pLimit; - pNew->pOffset = pOffset; pNew->pWith = 0; - assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 - || pParse->db->mallocFailed!=0 ); if( pParse->db->mallocFailed ) { clearSelect(pParse->db, pNew, pNew!=&standin); pNew = 0; @@ -1874,7 +1869,7 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ /* ** Compute the iLimit and iOffset fields of the SELECT based on the -** pLimit and pOffset expressions. pLimit and pOffset hold the expressions +** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions ** that appear in the original SQL statement after the LIMIT and OFFSET ** keywords. Or NULL if those keywords are omitted. iLimit and iOffset ** are the integer memory register numbers for counters used to compute @@ -1882,15 +1877,15 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ ** iLimit and iOffset are negative. ** ** This routine changes the values of iLimit and iOffset only if -** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values (zero) +** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit +** and iOffset should have been preset to appropriate default values (zero) ** prior to calling this routine. ** ** The iOffset register (if it exists) is initialized to the value ** of the OFFSET. The iLimit register is initialized to LIMIT. Register ** iOffset+1 is initialized to LIMIT+OFFSET. ** -** Only if pLimit!=0 or pOffset!=0 do the limit registers get +** Only if pLimit->pLeft!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple ** SELECT statements. @@ -1900,6 +1895,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ int iLimit = 0; int iOffset; int n; + Expr *pLimit = p->pLimit; + if( p->iLimit ) return; /* @@ -1909,12 +1906,13 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ ** no rows. */ sqlite3ExprCacheClear(pParse); - assert( p->pOffset==0 || p->pLimit!=0 ); - if( p->pLimit ){ + if( pLimit ){ + assert( pLimit->op==TK_LIMIT ); + assert( pLimit->pLeft!=0 ); p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(p->pLimit, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -1924,15 +1922,15 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->selFlags |= SF_FixedLimit; } }else{ - sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3ExprCode(pParse, pLimit->pLeft, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); } - if( p->pOffset ){ + if( pLimit->pRight ){ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ - sqlite3ExprCode(pParse, p->pOffset, iOffset); + sqlite3ExprCode(pParse, pLimit->pRight, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); @@ -2062,7 +2060,7 @@ static void generateWithRecursiveQuery( int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ - Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + Expr *pLimit; /* Saved LIMIT and OFFSET */ int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ /* Obtain authorization to do a recursive query */ @@ -2073,10 +2071,9 @@ static void generateWithRecursiveQuery( p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, addrBreak); pLimit = p->pLimit; - pOffset = p->pOffset; regLimit = p->iLimit; regOffset = p->iOffset; - p->pLimit = p->pOffset = 0; + p->pLimit = 0; p->iLimit = p->iOffset = 0; pOrderBy = p->pOrderBy; @@ -2169,7 +2166,6 @@ end_of_recursive_query: sqlite3ExprListDelete(pParse->db, p->pOrderBy); p->pOrderBy = pOrderBy; p->pLimit = pLimit; - p->pOffset = pOffset; return; } #endif /* SQLITE_OMIT_CTE */ @@ -2205,7 +2201,6 @@ static int multiSelectValues( assert( p->selFlags & SF_Values ); assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); assert( p->pLimit==0 ); - assert( p->pOffset==0 ); assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); if( p->pPrior==0 ) break; assert( p->pPrior->pNext==p ); @@ -2332,11 +2327,9 @@ static int multiSelect( pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - pPrior->pOffset = p->pOffset; explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &dest); p->pLimit = 0; - p->pOffset = 0; if( rc ){ goto multi_select_end; } @@ -2358,7 +2351,7 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit - && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) + && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit) && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); @@ -2373,7 +2366,7 @@ static int multiSelect( int unionTab; /* Cursor number of the temporary table holding result */ u8 op = 0; /* One of the SRT_ operations to apply to self */ int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ + Expr *pLimit; /* Saved values of p->nLimit */ int addr; SelectDest uniondest; @@ -2385,7 +2378,6 @@ static int multiSelect( ** right. */ assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - assert( p->pOffset==0 ); /* Not allowed on leftward elements */ unionTab = dest.iSDParm; }else{ /* We will need to create our own temporary table to hold the @@ -2421,8 +2413,6 @@ static int multiSelect( p->pPrior = 0; pLimit = p->pLimit; p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; uniondest.eDest = op; explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &uniondest); @@ -2438,7 +2428,6 @@ static int multiSelect( } sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; - p->pOffset = pOffset; p->iLimit = 0; p->iOffset = 0; @@ -2466,7 +2455,7 @@ static int multiSelect( default: assert( p->op==TK_INTERSECT ); { int tab1, tab2; int iCont, iBreak, iStart; - Expr *pLimit, *pOffset; + Expr *pLimit; int addr; SelectDest intersectdest; int r1; @@ -2502,8 +2491,6 @@ static int multiSelect( p->pPrior = 0; pLimit = p->pLimit; p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; intersectdest.iSDParm = tab2; explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &intersectdest); @@ -2513,7 +2500,6 @@ static int multiSelect( if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; - p->pOffset = pOffset; /* Generate code to take the intersection of the two temporary ** tables. @@ -2992,8 +2978,6 @@ static int multiSelectOrderBy( } sqlite3ExprDelete(db, p->pLimit); p->pLimit = 0; - sqlite3ExprDelete(db, p->pOffset); - p->pOffset = 0; regAddrA = ++pParse->nMem; regAddrB = ++pParse->nMem; @@ -3457,7 +3441,7 @@ static int flattenSubquery( ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ - if( pSub->pOffset ) return 0; /* Restriction (14) */ + if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */ if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ return 0; /* Restriction (15) */ } @@ -3605,16 +3589,13 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; - Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; p->pOrderBy = 0; p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; - p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); sqlite3SelectSetName(pNew, pSub->zSelName); - p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -4080,7 +4061,6 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; - pNew->pOffset = 0; return WRC_Continue; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 95a66b4fe7..058fee1461 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2749,7 +2749,6 @@ struct Select { Select *pPrior; /* Prior select in a compound select statement */ Select *pNext; /* Next select to the left in a compound */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ - Expr *pOffset; /* OFFSET expression. NULL means not used. */ With *pWith; /* WITH clause attached to this select. Or NULL. */ }; @@ -3756,16 +3755,16 @@ void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, void sqlite3DropIndex(Parse*, SrcList*, int); int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u32,Expr*,Expr*); + Expr*,ExprList*,u32,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, int); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) -Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); +Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); #endif -void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*); -void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,Expr*); +void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*); +void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*); WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); LogEst sqlite3WhereOutputRowCount(WhereInfo*); @@ -3889,7 +3888,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3*); void sqlite3ChangeCookie(Parse*, int); #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) -void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,Expr*,int); +void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/treeview.c b/src/treeview.c index ba9fa7b2f0..8d171a4733 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -153,7 +153,6 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ if( p->pHaving ) n++; if( p->pOrderBy ) n++; if( p->pLimit ) n++; - if( p->pOffset ) n++; } sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); if( p->pSrc && p->pSrc->nSrc ){ @@ -210,12 +209,12 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ } if( p->pLimit ){ sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); + if( p->pLimit->pRight ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); + sqlite3TreeViewPop(pView); + } sqlite3TreeViewPop(pView); } if( p->pPrior ){ diff --git a/src/trigger.c b/src/trigger.c index a17769ae98..a64fb291a0 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -711,7 +711,7 @@ static int codeTriggerProgram( targetSrcList(pParse, pStep), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), - pParse->eOrconf, 0, 0, 0 + pParse->eOrconf, 0, 0 ); break; } @@ -727,7 +727,7 @@ static int codeTriggerProgram( case TK_DELETE: { sqlite3DeleteFrom(pParse, targetSrcList(pParse, pStep), - sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0, 0 + sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); break; } diff --git a/src/update.c b/src/update.c index 4cfa4019eb..15e8f4a6ce 100644 --- a/src/update.c +++ b/src/update.c @@ -93,8 +93,7 @@ void sqlite3Update( Expr *pWhere, /* The WHERE clause. May be null */ int onError, /* How to handle constraint errors */ ExprList *pOrderBy, /* ORDER BY clause. May be null */ - Expr *pLimit, /* LIMIT clause. May be null */ - Expr *pOffset /* OFFSET clause. May be null */ + Expr *pLimit /* LIMIT clause. May be null */ ){ int i, j; /* Loop counters */ Table *pTab; /* The table to be updated */ @@ -182,10 +181,10 @@ void sqlite3Update( #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT if( !isView ){ pWhere = sqlite3LimitWhere( - pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "UPDATE" + pParse, pTabList, pWhere, pOrderBy, pLimit, "UPDATE" ); pOrderBy = 0; - pLimit = pOffset = 0; + pLimit = 0; } #endif @@ -358,10 +357,10 @@ void sqlite3Update( #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ sqlite3MaterializeView(pParse, pTab, - pWhere, pOrderBy, pLimit, pOffset, iDataCur + pWhere, pOrderBy, pLimit, iDataCur ); pOrderBy = 0; - pLimit = pOffset = 0; + pLimit = 0; } #endif @@ -748,7 +747,6 @@ update_cleanup: #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); - sqlite3ExprDelete(db, pOffset); #endif return; } diff --git a/src/walker.c b/src/walker.c index ae7545bf5f..60bf8226fe 100644 --- a/src/walker.c +++ b/src/walker.c @@ -91,7 +91,6 @@ int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort; return WRC_Continue; } From 35f51a4460f8548021ef1c709197622172a9e496 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Nov 2017 17:07:22 +0000 Subject: [PATCH 130/156] UI improvements on the .testctrl dot-command in the shell. FossilOrigin-Name: 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/shell.c.in | 59 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 1475fea8d1..959866cf88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sparse\stree,\scombine\sLIMIT\sand\sOFFSET\sinto\sa\ssingle\sexpression\srooted\non\sa\sTK_LIMIT\snode,\sfor\sa\ssmall\scode\ssize\sreduction\sand\sperformance\sincrease,\nand\sa\sreduction\sin\scode\scomplexity. -D 2017-11-14T23:48:23.099 +C UI\simprovements\son\sthe\s.testctrl\sdot-command\sin\sthe\sshell. +D 2017-11-15T17:07:22.982 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a +F src/shell.c.in e261e3c2f39edbabfdea00ef8f50adf7b2fd7d84fc6d78d8c2a06b335bd1916f F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3 -R 9fe4e3d4453a36374c8a196662e2c5b0 +P 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826 +R 837aef86d6c701c4be924bf38f742114 U drh -Z 3ab594fe26a5b432ce6d9e25b1b1040f +Z 657eeee0b34954f0ac1c7bdc4eb3dc1b diff --git a/manifest.uuid b/manifest.uuid index 41cb7f1619..7a1cc65c2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826 \ No newline at end of file +5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e85ebd3c93..1b5adc0734 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5922,49 +5922,68 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_UNTESTABLE - if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){ + if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ } aCtrl[] = { - { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE }, - { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE }, - { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET }, - { "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST }, - { "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL }, - { "benign_malloc_hooks", SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS }, - { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE }, - { "assert", SQLITE_TESTCTRL_ASSERT }, { "always", SQLITE_TESTCTRL_ALWAYS }, - { "reserve", SQLITE_TESTCTRL_RESERVE }, - { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, - { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, + { "assert", SQLITE_TESTCTRL_ASSERT }, + { "benign_malloc_hooks", SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS }, + { "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST }, { "byteorder", SQLITE_TESTCTRL_BYTEORDER }, - { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT }, + { "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL }, { "imposter", SQLITE_TESTCTRL_IMPOSTER }, + { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, + { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT }, + { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, + { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE }, + { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET }, + { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE }, + { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE }, + { "reserve", SQLITE_TESTCTRL_RESERVE }, }; int testctrl = -1; int rc2 = 0; int i, n2; open_db(p, 0); + const char *zCmd = nArg>=2 ? azArg[1] : "help"; + + /* The argument can optionally begin with "-" or "--" */ + if( zCmd[0]=='-' && zCmd[1] ){ + zCmd++; + if( zCmd[0]=='-' && zCmd[1] ) zCmd++; + } + + /* --help lists all test-controls */ + if( strcmp(zCmd,"help")==0 ){ + utf8_printf(p->out, "Available test-controls:\n"); + for(i=0; iout, " .testctrl %s\n", aCtrl[i].zCtrlName); + } + rc = 1; + goto meta_command_exit; + } /* convert testctrl text option to value. allow any unique prefix ** of the option name, or a numerical value. */ - n2 = strlen30(azArg[1]); + n2 = strlen30(zCmd); for(i=0; iSQLITE_TESTCTRL_LAST) ){ - utf8_printf(stderr,"Error: invalid testctrl option: %s\n", azArg[1]); + utf8_printf(stderr,"Error: unknown test-control: %s\n" + "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ From ef302e8a676b42f061aadcf1e12c9f9bceb53570 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Nov 2017 19:14:08 +0000 Subject: [PATCH 131/156] Futher improvements to the .testctrl interface in the shell. FossilOrigin-Name: 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d --- manifest | 12 +++--- manifest.uuid | 2 +- src/shell.c.in | 101 ++++++++++++++++++++++++------------------------- 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 959866cf88..31f6458b7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C UI\simprovements\son\sthe\s.testctrl\sdot-command\sin\sthe\sshell. -D 2017-11-15T17:07:22.982 +C Futher\simprovements\sto\sthe\s.testctrl\sinterface\sin\sthe\sshell. +D 2017-11-15T19:14:08.134 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in e261e3c2f39edbabfdea00ef8f50adf7b2fd7d84fc6d78d8c2a06b335bd1916f +F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a909 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826 -R 837aef86d6c701c4be924bf38f742114 +P 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5 +R bb554ed3e0e111141682146f45553c3a U drh -Z 657eeee0b34954f0ac1c7bdc4eb3dc1b +Z e5620a9dcff0b704be870774526d47b0 diff --git a/manifest.uuid b/manifest.uuid index 7a1cc65c2e..659585bc39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5 \ No newline at end of file +1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1b5adc0734..68a337db11 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5926,25 +5926,31 @@ static int do_meta_command(char *zLine, ShellState *p){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ + const char *zUsage; /* Usage notes */ } aCtrl[] = { - { "always", SQLITE_TESTCTRL_ALWAYS }, - { "assert", SQLITE_TESTCTRL_ASSERT }, - { "benign_malloc_hooks", SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS }, - { "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST }, - { "byteorder", SQLITE_TESTCTRL_BYTEORDER }, - { "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL }, - { "imposter", SQLITE_TESTCTRL_IMPOSTER }, - { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, - { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT }, - { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, - { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE }, - { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET }, - { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE }, - { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE }, - { "reserve", SQLITE_TESTCTRL_RESERVE }, + { "always", SQLITE_TESTCTRL_ALWAYS, "BOOLEAN" }, + { "assert", SQLITE_TESTCTRL_ASSERT, "BOOLEAN" }, + /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, "" },*/ + /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, "" },*/ + { "byteorder", SQLITE_TESTCTRL_BYTEORDER, "" }, + /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, "" }, */ + { "imposter", SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"}, +#ifdef SQLITE_N_KEYWORD + { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD, "IDENTIFIER" }, +#endif + { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, + { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, + { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, + { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, + { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET, "" }, + { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, + { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, + { "reserve", SQLITE_TESTCTRL_RESERVE, "BYTES-OF-RESERVE" }, }; int testctrl = -1; - int rc2 = 0; + int iCtrl = -1; + int rc2 = 0; /* 0: usage. 1: %d 2: %x 3: no-output */ + int isOk = 0; int i, n2; open_db(p, 0); const char *zCmd = nArg>=2 ? azArg[1] : "help"; @@ -5959,7 +5965,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if( strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available test-controls:\n"); for(i=0; iout, " .testctrl %s\n", aCtrl[i].zCtrlName); + utf8_printf(p->out, " .testctrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; goto meta_command_exit; @@ -5972,6 +5979,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){ if( testctrl<0 ){ testctrl = aCtrl[i].ctrlCode; + iCtrl = i; }else{ utf8_printf(stderr, "Error: ambiguous test-control: \"%s\"\n" "Use \".testctrl --help\" for help\n", zCmd); @@ -5980,8 +5988,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } } - if( testctrl<0 ) testctrl = (int)integerValue(zCmd); - if( (testctrlSQLITE_TESTCTRL_LAST) ){ + if( testctrl<0 ){ utf8_printf(stderr,"Error: unknown test-control: %s\n" "Use \".testctrl --help\" for help\n", zCmd); }else{ @@ -5993,10 +6000,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ int opt = (int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - } else { - utf8_printf(stderr,"Error: testctrl %s takes a single int option\n", - azArg[1]); + isOk = 3; } break; @@ -6007,10 +6011,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_BYTEORDER: if( nArg==2 ){ rc2 = sqlite3_test_control(testctrl); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - } else { - utf8_printf(stderr,"Error: testctrl %s takes no options\n", - azArg[1]); + isOk = testctrl==SQLITE_TESTCTRL_BYTEORDER ? 1 : 3; } break; @@ -6019,24 +6020,27 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ unsigned int opt = (unsigned int)integerValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - } else { - utf8_printf(stderr,"Error: testctrl %s takes a single unsigned" - " int option\n", azArg[1]); + isOk = 3; } break; /* sqlite3_test_control(int, int) */ case SQLITE_TESTCTRL_ASSERT: case SQLITE_TESTCTRL_ALWAYS: + if( nArg==3 ){ + int opt = booleanValue(azArg[2]); + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 1; + } + break; + + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: case SQLITE_TESTCTRL_NEVER_CORRUPT: if( nArg==3 ){ int opt = booleanValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - } else { - utf8_printf(stderr,"Error: testctrl %s takes a single int option\n", - azArg[1]); + isOk = 3; } break; @@ -6046,11 +6050,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ const char *opt = azArg[2]; rc2 = sqlite3_test_control(testctrl, opt); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - } else { - utf8_printf(stderr, - "Error: testctrl %s takes a single char * option\n", - azArg[1]); + isOk = 1; } break; #endif @@ -6061,22 +6061,19 @@ static int do_meta_command(char *zLine, ShellState *p){ azArg[2], integerValue(azArg[3]), integerValue(azArg[4])); - raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); - }else{ - raw_printf(stderr,"Usage: .testctrl imposter dbName onoff tnum\n"); + isOk = 3; } break; - - case SQLITE_TESTCTRL_BITVEC_TEST: - case SQLITE_TESTCTRL_FAULT_INSTALL: - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: - default: - utf8_printf(stderr, - "Error: CLI support for testctrl %s not implemented\n", - azArg[1]); - break; } } + if( isOk==0 && iCtrl>=0 ){ + utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); + rc = 1; + }else if( isOk==1 ){ + raw_printf(p->out, "%d\n", rc2); + }else if( isOk==2 ){ + raw_printf(p->out, "0x%08x\n", rc2); + } }else #endif /* !defined(SQLITE_UNTESTABLE) */ From 3dbb4acf0715336c867758d6de81e57a2a1b8e96 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Nov 2017 19:04:33 +0000 Subject: [PATCH 132/156] Add a missing "finish_test" to the end of the stmtvtab1.test script. FossilOrigin-Name: e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/stmtvtab1.test | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 31f6458b7c..df822c015e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Futher\simprovements\sto\sthe\s.testctrl\sinterface\sin\sthe\sshell. -D 2017-11-15T19:14:08.134 +C Add\sa\smissing\s"finish_test"\sto\sthe\send\sof\sthe\sstmtvtab1.test\sscript. +D 2017-11-16T19:04:33.787 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -1249,7 +1249,7 @@ F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f -F test/stmtvtab1.test acc3c40f484f2c4922e270724383d715abb9d69676da907a9c64f31c54f2ef9f +F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5 -R bb554ed3e0e111141682146f45553c3a +P 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d +R 43f6376592b4d26341e49684bbb727c1 U drh -Z e5620a9dcff0b704be870774526d47b0 +Z 7e47c726c44355ca56f3a93b39a4aaed diff --git a/manifest.uuid b/manifest.uuid index 659585bc39..4216e3c295 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d \ No newline at end of file +e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 \ No newline at end of file diff --git a/test/stmtvtab1.test b/test/stmtvtab1.test index 4937102091..e8a79fdacd 100644 --- a/test/stmtvtab1.test +++ b/test/stmtvtab1.test @@ -78,3 +78,5 @@ db cache flush do_execsql_test stmtvtab1-160 { SELECT * FROM sqlite_stmt WHERE NOT busy; } {} + +finish_test From cda185d3a268f17faf85968b036070d45bd030fb Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Nov 2017 20:48:47 +0000 Subject: [PATCH 133/156] Ensure that empty space on pages is zeroed before they are written to disk. This helps with compressed databases, and stops valgrind complaining about uninitialized bytes and write(). FossilOrigin-Name: 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 --- ext/lsm1/lsm_sorted.c | 66 +++++++++++++++++++++++++++---------------- manifest | 14 ++++----- manifest.uuid | 2 +- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c index f479f4ce8c..9eff2d08cc 100644 --- a/ext/lsm1/lsm_sorted.c +++ b/ext/lsm1/lsm_sorted.c @@ -92,7 +92,7 @@ #define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry) +#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) #define SEGMENT_BTREE_FLAG 0x0001 #define PGFTR_SKIP_NEXT_FLAG 0x0002 @@ -3999,6 +3999,11 @@ static int mergeWorkerWrite( ** marked read-only, advance to the next page of the output run. */ iOff = pMerge->iOutputOff; if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ + if( iOff>=0 && pPg ){ + /* Zero any free space on the page */ + assert( aData ); + memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); + } iFPtr = (int)*pMW->pCsr->pPrevMergePtr; iRPtr = iPtr - iFPtr; iOff = 0; @@ -4069,36 +4074,49 @@ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ /* Unless the merge has finished, save the cursor position in the ** Merge.aInput[] array. See function mergeWorkerInit() for the ** code to restore a cursor position based on aInput[]. */ - if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){ + if( rc==LSM_OK && pCsr ){ Merge *pMerge = pMW->pLevel->pMerge; - int bBtree = (pCsr->pBtCsr!=0); - int iPtr; + if( lsmMCursorValid(pCsr) ){ + int bBtree = (pCsr->pBtCsr!=0); + int iPtr; - /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ - assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); - assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); + /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ + assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); + assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); - for(i=0; i<(pMerge->nInput-bBtree); i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - if( pPtr->pPg ){ - pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); - pMerge->aInput[i].iCell = pPtr->iCell; + for(i=0; i<(pMerge->nInput-bBtree); i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + if( pPtr->pPg ){ + pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); + pMerge->aInput[i].iCell = pPtr->iCell; + }else{ + pMerge->aInput[i].iPg = 0; + pMerge->aInput[i].iCell = 0; + } + } + if( bBtree && pMerge->nInput ){ + assert( i==pCsr->nPtr ); + btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); + } + + /* Store the location of the split-key */ + iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; + if( iPtrnPtr ){ + pMerge->splitkey = pMerge->aInput[iPtr]; }else{ - pMerge->aInput[i].iPg = 0; - pMerge->aInput[i].iCell = 0; + btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); } } - if( bBtree && pMerge->nInput ){ - assert( i==pCsr->nPtr ); - btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); - } - /* Store the location of the split-key */ - iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - pMerge->splitkey = pMerge->aInput[iPtr]; - }else{ - btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); + /* Zero any free space left on the final page. This helps with + ** compression if using a compression hook. And prevents valgrind + ** from complaining about uninitialized byte passed to write(). */ + if( pMW->pPage ){ + int nData; + u8 *aData = fsPageData(pMW->pPage, &nData); + int iOff = pMerge->iOutputOff; + int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); + memset(&aData[iOff], 0, iEof - iOff); } pMerge->iOutputOff = -1; diff --git a/manifest b/manifest index df822c015e..d601661e29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\s"finish_test"\sto\sthe\send\sof\sthe\sstmtvtab1.test\sscript. -D 2017-11-16T19:04:33.787 +C Ensure\sthat\sempty\sspace\son\spages\sis\szeroed\sbefore\sthey\sare\swritten\sto\sdisk.\nThis\shelps\swith\scompressed\sdatabases,\sand\sstops\svalgrind\scomplaining\sabout\nuninitialized\sbytes\sand\swrite(). +D 2017-11-16T20:48:47.980 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -246,7 +246,7 @@ F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525 -F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9 +F ext/lsm1/lsm_sorted.c 8f899fb64a4c736ff3c27d5126c7ce181129ddffde947fe5fb657a7a413f470b F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d -R 43f6376592b4d26341e49684bbb727c1 -U drh -Z 7e47c726c44355ca56f3a93b39a4aaed +P e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 +R a54da776e28f1875d13b67b86731a30b +U dan +Z 4bb1ae38bdfadd3dc2e110f3406150e8 diff --git a/manifest.uuid b/manifest.uuid index 4216e3c295..39825a308f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 \ No newline at end of file +5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 \ No newline at end of file From 6cbc5074e8ff6ec0b4953573c3cbfe015ba1f779 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Nov 2017 08:20:10 +0000 Subject: [PATCH 134/156] Avoid creating a master journal if all or all but one of the databases in the transaction is a temp file. FossilOrigin-Name: 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508 --- manifest | 17 +++++----- manifest.uuid | 2 +- src/vdbeaux.c | 1 + test/mjournal.test | 77 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d601661e29..8fa35294e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sempty\sspace\son\spages\sis\szeroed\sbefore\sthey\sare\swritten\sto\sdisk.\nThis\shelps\swith\scompressed\sdatabases,\sand\sstops\svalgrind\scomplaining\sabout\nuninitialized\sbytes\sand\swrite(). -D 2017-11-16T20:48:47.980 +C Avoid\screating\sa\smaster\sjournal\sif\sall\sor\sall\sbut\sone\sof\sthe\sdatabases\sin\sthe\ntransaction\sis\sa\stemp\sfile. +D 2017-11-17T08:20:10.452 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -543,7 +543,7 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 -F src/vdbeaux.c 9521a9364e68edad3c8d05ae63395d076724bed1c878c9b13fab61ada54e7d2a +F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f @@ -1062,7 +1062,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7 -F test/mjournal.test 68b749956f9a179e7e633a3958b48a5a905d28d30c7ec88f3f26dc6f220129db +F test/mjournal.test 946d7161cabbc45ecc818d7fa38c1925bbf36e5534a08a81fa0203ee7ced7d54 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e @@ -1677,7 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 -R a54da776e28f1875d13b67b86731a30b +P 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 +R 549acd152a27430f44fc7371aab0d342 +T *branch * master-journal-temp-files +T *sym-master-journal-temp-files * +T -sym-trunk * U dan -Z 4bb1ae38bdfadd3dc2e110f3406150e8 +Z adbdc7db93ebd9a4eb2724ff9604ae84 diff --git a/manifest.uuid b/manifest.uuid index 39825a308f..52e9077fed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 \ No newline at end of file +355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e057a5df5f..e73a339f81 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2268,6 +2268,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ pPager = sqlite3BtreePager(pBt); if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] + && sqlite3PagerIsMemdb(pPager)==0 ){ assert( i!=1 ); nTrans++; diff --git a/test/mjournal.test b/test/mjournal.test index aab2c08b51..cf6ef5f7b5 100644 --- a/test/mjournal.test +++ b/test/mjournal.test @@ -79,5 +79,80 @@ do_execsql_test 1.6 { SELECT * FROM t1; } - +#------------------------------------------------------------------------- +# Check that master journals are not created if the transaction involves +# multiple temp files. +# +db close +testvfs tvfs +tvfs filter xOpen +tvfs script open_cb +set ::open "" +proc open_cb {method file arglist} { + lappend ::open $file +} + +proc contains_mj {} { + foreach f $::open { + set t [file tail $f] + if {[string match *mj* $t]} { return 1 } + } + return 0 +} + +# Like [do_execsql_test], except that a boolean indicating whether or +# not a master journal file was opened ([file tail] contains "mj") or +# not. Example: +# +# do_hasmj_test 1.0 { SELECT 'a', 'b' } {0 a b} +# +proc do_hasmj_test {tn sql expected} { + set ::open [list] + uplevel [list do_test $tn [subst -nocommands { + set res [execsql "$sql"] + concat [contains_mj] [set res] + }] [list {*}$expected]] +} + +forcedelete test.db +forcedelete test.db2 +forcedelete test.db3 +sqlite3 db test.db -vfs tvfs + +do_execsql_test 2.0 { + ATTACH 'test.db2' AS dbfile; + ATTACH '' AS dbtemp; + ATTACH ':memory:' AS dbmem; + + CREATE TABLE t1(x); + CREATE TABLE dbfile.t2(x); + CREATE TABLE dbtemp.t3(x); + CREATE TABLE dbmem.t4(x); +} + +# Two real files. +do_hasmj_test 2.1 { + BEGIN; + INSERT INTO t1 VALUES(1); + INSERT INTO t2 VALUES(1); + COMMIT; +} {1} + +# One real, one temp file. +do_hasmj_test 2.2 { + BEGIN; + INSERT INTO t1 VALUES(1); + INSERT INTO t3 VALUES(1); + COMMIT; +} {0} + +# One file, one :memory: db. +do_hasmj_test 2.3 { + BEGIN; + INSERT INTO t1 VALUES(1); + INSERT INTO t4 VALUES(1); + COMMIT; +} {0} + finish_test + From 7fc0ba0f4cb864cfb518aae35b6fb4adbe7a12f4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Nov 2017 15:02:00 +0000 Subject: [PATCH 135/156] Clarification of comments on sqlite3FindInIndex(). No changes to code. FossilOrigin-Name: 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/expr.c | 13 ++++++------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 103854a9cb..a59ee6a275 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\scount\stemporary\sdatabases\sthat\shave\sbeen\sattached\susing\sATTACH\swhen\nfiguring\sout\sif\sa\smaster-journal\sfile\sis\srequired\sby\sa\stransaction. -D 2017-11-17T13:21:12.384 +C Clarification\sof\scomments\son\ssqlite3FindInIndex().\s\sNo\schanges\sto\scode. +D 2017-11-17T15:02:00.058 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -423,7 +423,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3 -F src/expr.c 5257a9157f22f048ddcce5cd494d39633e89c2a4769671311b3e7875d262f746 +F src/expr.c fe11b91bb65b869143bd42023427c4429778ae42c0a0db7762f68f75b347a958 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13 @@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508 -R 549acd152a27430f44fc7371aab0d342 -T +closed 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508 -U dan -Z eb504c2f876fdf46c1053782d9bac493 +P 93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0 +R 9e3518c1c08ebf61892be3d9ca3ce745 +U drh +Z 183368b52c0a93e109b0b85758c70894 diff --git a/manifest.uuid b/manifest.uuid index 39dbcda631..cea62a5a5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0 \ No newline at end of file +071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 5027f994db..524e539349 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2186,16 +2186,15 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** pX->iTable made to point to the ephemeral table instead of an ** existing table. ** -** The inFlags parameter must contain exactly one of the bits -** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains -** IN_INDEX_MEMBERSHIP, then the generated table will be used for a -** fast membership test. When the IN_INDEX_LOOP bit is set, the -** IN index will be used to loop over all values of the RHS of the -** IN operator. +** The inFlags parameter must contain, at a minimum, one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a fast +** membership test. When the IN_INDEX_LOOP bit is set, the IN index will +** be used to loop over all values of the RHS of the IN operator. ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected columns are guaranteed +** An epheremal table will be created unless the selected columns are guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or due to ** a UNIQUE constraint or index. ** From 9b1ecb67c3a5bd5a049b4e2c0d027405e991febd Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Nov 2017 17:32:40 +0000 Subject: [PATCH 136/156] New assert() statements in the rowvalue IN expression processing. FossilOrigin-Name: 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wherecode.c | 29 +++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a59ee6a275..860e60f5e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarification\sof\scomments\son\ssqlite3FindInIndex().\s\sNo\schanges\sto\scode. -D 2017-11-17T15:02:00.058 +C New\sassert()\sstatements\sin\sthe\srowvalue\sIN\sexpression\sprocessing. +D 2017-11-17T17:32:40.141 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -555,7 +555,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 -F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 +F src/wherecode.c 8605c0ca0c34d4692011cf68a5f4cfc85352c1df917dc6eada320cecc4f5ea73 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0 -R 9e3518c1c08ebf61892be3d9ca3ce745 +P 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace +R e07fbdd249f84df3df0af0935ae0894c U drh -Z 183368b52c0a93e109b0b85758c70894 +Z dfffcc7482c8c9b6b4a114214fd236e3 diff --git a/manifest.uuid b/manifest.uuid index cea62a5a5c..e60558e452 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace \ No newline at end of file +00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index da5c686a95..cc2759eeaf 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -377,6 +377,27 @@ static void updateRangeAffinityStr( } } +#ifdef SQLITE_DEBUG +/* Return true if the pSub ExprList is a subset of pMain. The terms +** of pSub can be in a different order from pMain. The only requirement +** is that every term in pSub must exist somewhere in pMain. +** +** Return false if pSub contains any term that is not found in pMain. +*/ +static int exprListSubset(ExprList *pSub, ExprList *pMain){ + int i, j; + for(i=0; inExpr; i++){ + Expr *p = pSub->a[i].pExpr; + for(j=0; jnExpr; j++){ + if( sqlite3ExprCompare(0, p, pMain->a[j].pExpr, 0)==0 ) break; + } + if( j>=pMain->nExpr ) return 0; + } + return 1; +} +#endif /* SQLITE_DEBUG */ + + /* ** Generate code for a single equality term of the WHERE clause. An equality ** term can be either X=expr or X IN (...). pTerm is the term to be @@ -463,6 +484,14 @@ static int codeEqualityTerm( pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs); } } + + /* pRhs should be a subset of pOrigRhs (though possibly in a different + ** order). And pLhs should be a subset of pOrigLhs. To put it + ** another way: Every term of pRhs should exist in pOrigRhs and + ** every term of pLhs should exist in pOrigLhs. */ + assert( db->mallocFailed || exprListSubset(pRhs, pOrigRhs) ); + assert( db->mallocFailed || exprListSubset(pLhs, pOrigLhs) ); + if( !db->mallocFailed ){ Expr *pLeft = pX->pLeft; From 6bcabfe1a38ae090c276bb99ff73c8f6e4e70b3d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Nov 2017 20:07:19 +0000 Subject: [PATCH 137/156] Add some missing "finish_test" lines to the end of test scripts. FossilOrigin-Name: c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- test/tkt-26ff0c2d1e.test | 2 ++ test/tkt-7a31705a7e6.test | 2 ++ test/tkt-a8a0d2996a.test | 2 ++ test/tkt3334.test | 2 ++ test/vacuum4.test | 2 ++ test/varint.test | 2 ++ 8 files changed, 25 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 860e60f5e2..5615a01c67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sassert()\sstatements\sin\sthe\srowvalue\sIN\sexpression\sprocessing. -D 2017-11-17T17:32:40.141 +C Add\ssome\smissing\s"finish_test"\slines\sto\sthe\send\sof\stest\sscripts. +D 2017-11-17T20:07:19.718 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -1292,7 +1292,7 @@ F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c -F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 +F test/tkt-26ff0c2d1e.test c15bec890c4d226c0da2f35ff30f9e84c169cfef90e73a8cb5cec11d723dfa96 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 @@ -1314,7 +1314,7 @@ F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf F test/tkt-78e04e52ea.test 1b2e6bf4f1d9887b216b6da774e5f25915ec8118 -F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 +F test/tkt-7a31705a7e6.test 9e9c057b6a9497c8f7ba7b16871029414ccf6550e7345d9085d6d71c9a56bb6f F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 F test/tkt-80e031a00f.test 9ee36348b761bf7c14261e002b75a4c0d5a04d4c @@ -1327,7 +1327,7 @@ F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 -F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 +F test/tkt-a8a0d2996a.test 76662ff0622c90e7ce7bbcb4d9e1129acddf877d17c3489f2da7f17ddfaad1f4 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 @@ -1391,7 +1391,7 @@ F test/tkt3121.test 536df66a02838c26a12fe98639354ca1290ca68b F test/tkt3201.test f1500ccecc0d578dc4cde7d3242008297c4d59b3 F test/tkt3292.test 962465a0984a3b8c757efe59c2c59144871ee1dd F test/tkt3298.test 20fd8773b825cb602e033aa04f8602e1ebdcd93c -F test/tkt3334.test ea13a53cb176e90571a76c86605b14a09efe366d +F test/tkt3334.test 9756631e3c4aa3c416362c279e3c0953a83b7ca8274cb81a13264bb56296d8b0 F test/tkt3346.test 6f67c3ed7db94dfc5df4f5f0b63809a1f611e01a F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b @@ -1479,10 +1479,10 @@ F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d -F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 +F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010 F test/vacuum5.test c87234e8ca4107f349da4edbeda3e4ea5adc93f3 F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 -F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 +F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab F test/vtab1.test 8f91b9538d1404c3932293a588c4344218a0c94792d4289bb55e41020e7b3fff @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace -R e07fbdd249f84df3df0af0935ae0894c -U drh -Z dfffcc7482c8c9b6b4a114214fd236e3 +P 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c +R 7da8d3fa9a019a6fbc5c0bb38be51e2d +U dan +Z 96ccfa676c7df4dbf371fdb6ce8e1bc3 diff --git a/manifest.uuid b/manifest.uuid index e60558e452..de5b268438 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c \ No newline at end of file +c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26 \ No newline at end of file diff --git a/test/tkt-26ff0c2d1e.test b/test/tkt-26ff0c2d1e.test index 83a4f3d674..d7c5c6bae3 100644 --- a/test/tkt-26ff0c2d1e.test +++ b/test/tkt-26ff0c2d1e.test @@ -31,3 +31,5 @@ do_test bug-20100512-3 { sqlite3_column_int $STMT 0 } {555} sqlite3_finalize $STMT + +finish_test diff --git a/test/tkt-7a31705a7e6.test b/test/tkt-7a31705a7e6.test index e3e402917d..fd862484ea 100644 --- a/test/tkt-7a31705a7e6.test +++ b/test/tkt-7a31705a7e6.test @@ -23,3 +23,5 @@ do_execsql_test tkt-7a31705a7e6-1.1 { CREATE TABLE t2x (b INTEGER PRIMARY KEY); SELECT t1.a FROM ((t1 JOIN t2 ON t1.a=t2.a) AS x JOIN t2x ON x.b=t2x.b) as y; } {} + +finish_test diff --git a/test/tkt-a8a0d2996a.test b/test/tkt-a8a0d2996a.test index 6b15e410e7..03c2ee9448 100644 --- a/test/tkt-a8a0d2996a.test +++ b/test/tkt-a8a0d2996a.test @@ -91,3 +91,5 @@ do_execsql_test 4.5 { do_execsql_test 4.6 { SELECT '1234x'/'10y'; } {123.4} + +finish_test diff --git a/test/tkt3334.test b/test/tkt3334.test index 5473ab4cf8..3527932de3 100644 --- a/test/tkt3334.test +++ b/test/tkt3334.test @@ -82,3 +82,5 @@ do_test tkt3334-1.10 { SELECT count(*) FROM (SELECT a FROM t1) WHERE a=1; } } {3} + +finish_test diff --git a/test/vacuum4.test b/test/vacuum4.test index 326d037276..bbf0dd4379 100644 --- a/test/vacuum4.test +++ b/test/vacuum4.test @@ -65,3 +65,5 @@ do_test vacuum4-1.1 { VACUUM; } } {} + +finish_test diff --git a/test/varint.test b/test/varint.test index 974e88f2a6..fd0ec0d415 100644 --- a/test/varint.test +++ b/test/varint.test @@ -30,3 +30,5 @@ foreach start {0 100 10000 1000000 0x10000000} { } } } + +finish_test From 2410243e9d27b5f1f5862368eb52ded8898250d8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Nov 2017 21:01:04 +0000 Subject: [PATCH 138/156] Improved fix for ticket [da78413751863] that does not require disabling the query flattener as was done in [005d5b870625]. This also makes the code generator for vector IN operators a little easier to understand. FossilOrigin-Name: 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f --- manifest | 14 ++-- manifest.uuid | 2 +- src/wherecode.c | 170 +++++++++++++++++++++++++++--------------------- 3 files changed, 104 insertions(+), 82 deletions(-) diff --git a/manifest b/manifest index 5615a01c67..b8aae7e011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\smissing\s"finish_test"\slines\sto\sthe\send\sof\stest\sscripts. -D 2017-11-17T20:07:19.718 +C Improved\sfix\sfor\sticket\s[da78413751863]\sthat\sdoes\snot\srequire\sdisabling\sthe\nquery\sflattener\sas\swas\sdone\sin\s[005d5b870625].\s\sThis\salso\smakes\sthe\scode\ngenerator\sfor\svector\sIN\soperators\sa\slittle\seasier\sto\sunderstand. +D 2017-11-17T21:01:04.980 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -555,7 +555,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 -F src/wherecode.c 8605c0ca0c34d4692011cf68a5f4cfc85352c1df917dc6eada320cecc4f5ea73 +F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c -R 7da8d3fa9a019a6fbc5c0bb38be51e2d -U dan -Z 96ccfa676c7df4dbf371fdb6ce8e1bc3 +P c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26 +R 04b5e48559de23ef7902e501e8dcaac8 +U drh +Z cc87a690f1d098ebb337ab16a1f38777 diff --git a/manifest.uuid b/manifest.uuid index de5b268438..99c0990a4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26 \ No newline at end of file +723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index cc2759eeaf..40c5f41182 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -377,25 +377,100 @@ static void updateRangeAffinityStr( } } -#ifdef SQLITE_DEBUG -/* Return true if the pSub ExprList is a subset of pMain. The terms -** of pSub can be in a different order from pMain. The only requirement -** is that every term in pSub must exist somewhere in pMain. + +/* +** pX is an expression of the form: (vector) IN (SELECT ...) +** In other words, it is a vector IN operator with a SELECT clause on the +** LHS. But not all terms in the vector are indexable and the terms might +** not be in the correct order for indexing. ** -** Return false if pSub contains any term that is not found in pMain. +** This routine makes a copy of the input pX expression and then adjusts +** the vector on the LHS with corresponding changes to the SELECT so that +** the vector contains only index terms and those terms are in the correct +** order. The modified IN expression is returned. The caller is responsible +** for deleting the returned expression. +** +** Example: +** +** CREATE TABLE t1(a,b,c,d,e,f); +** CREATE INDEX t1x1 ON t1(e,c); +** SELECT * FROM t1 WHERE (a,b,c,d,e) IN (SELECT v,w,x,y,z FROM t2) +** \_______________________________________/ +** The pX expression +** +** Since only columns e and c can be used with the index, in that order, +** the modified IN expression that is returned will be: +** +** (e,c) IN (SELECT z,x FROM t2) +** +** The reduced pX is different from the original (obviously) and thus is +** only used for indexing, to improve performance. The original unaltered +** IN expression must also be run on each output row for correctness. */ -static int exprListSubset(ExprList *pSub, ExprList *pMain){ - int i, j; - for(i=0; inExpr; i++){ - Expr *p = pSub->a[i].pExpr; - for(j=0; jnExpr; j++){ - if( sqlite3ExprCompare(0, p, pMain->a[j].pExpr, 0)==0 ) break; +static Expr *removeUnindexableInClauseTerms( + Parse *pParse, /* The parsing context */ + int iEq, /* Look at loop terms starting here */ + WhereLoop *pLoop, /* The current loop */ + Expr *pX /* The IN expression to be reduced */ +){ + sqlite3 *db = pParse->db; + Expr *pNew = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed==0 ){ + ExprList *pOrigRhs = pNew->x.pSelect->pEList; /* Original unmodified RHS */ + ExprList *pOrigLhs = pNew->pLeft->x.pList; /* Original unmodified LHS */ + ExprList *pRhs = 0; /* New RHS after modifications */ + ExprList *pLhs = 0; /* New LHS after mods */ + int i; /* Loop counter */ + Select *pSelect; /* Pointer to the SELECT on the RHS */ + + for(i=iEq; inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField = pLoop->aLTerm[i]->iField - 1; + assert( pOrigRhs->a[iField].pExpr!=0 ); + pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); + pOrigRhs->a[iField].pExpr = 0; + assert( pOrigLhs->a[iField].pExpr!=0 ); + pLhs = sqlite3ExprListAppend(pParse, pLhs, pOrigLhs->a[iField].pExpr); + pOrigLhs->a[iField].pExpr = 0; + } } - if( j>=pMain->nExpr ) return 0; + sqlite3ExprListDelete(db, pOrigRhs); + sqlite3ExprListDelete(db, pOrigLhs); + pNew->pLeft->x.pList = pLhs; + pNew->x.pSelect->pEList = pRhs; + if( pLhs && pLhs->nExpr==1 ){ + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + Expr *p = pLhs->a[0].pExpr; + pLhs->a[0].pExpr = 0; + sqlite3ExprDelete(db, pNew->pLeft); + pNew->pLeft = p; + } + pSelect = pNew->x.pSelect; + if( pSelect->pOrderBy ){ + /* If the SELECT statement has an ORDER BY clause, zero the + ** iOrderByCol variables. These are set to non-zero when an + ** ORDER BY term exactly matches one of the terms of the + ** result-set. Since the result-set of the SELECT statement may + ** have been modified or reordered, these variables are no longer + ** set correctly. Since setting them is just an optimization, + ** it's easiest just to zero them here. */ + ExprList *pOrderBy = pSelect->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + } + +#if 0 + printf("For indexing, change the IN expr:\n"); + sqlite3TreeViewExpr(0, pX, 0); + printf("Into:\n"); + sqlite3TreeViewExpr(0, pNew, 0); +#endif } - return 1; + return pNew; } -#endif /* SQLITE_DEBUG */ /* @@ -460,76 +535,23 @@ static int codeEqualityTerm( } } for(i=iEq;inLTerm; i++){ - if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++; + assert( pLoop->aLTerm[i]!=0 ); + if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; } if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){ eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0); }else{ - Select *pSelect = pX->x.pSelect; sqlite3 *db = pParse->db; - u16 savedDbOptFlags = db->dbOptFlags; - ExprList *pOrigRhs = pSelect->pEList; - ExprList *pOrigLhs = pX->pLeft->x.pList; - ExprList *pRhs = 0; /* New Select.pEList for RHS */ - ExprList *pLhs = 0; /* New pX->pLeft vector */ - - for(i=iEq;inLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iField = pLoop->aLTerm[i]->iField - 1; - Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0); - Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0); - - pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs); - pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs); - } - } - - /* pRhs should be a subset of pOrigRhs (though possibly in a different - ** order). And pLhs should be a subset of pOrigLhs. To put it - ** another way: Every term of pRhs should exist in pOrigRhs and - ** every term of pLhs should exist in pOrigLhs. */ - assert( db->mallocFailed || exprListSubset(pRhs, pOrigRhs) ); - assert( db->mallocFailed || exprListSubset(pLhs, pOrigLhs) ); + pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); if( !db->mallocFailed ){ - Expr *pLeft = pX->pLeft; - - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; inExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; - } - } - - /* Take care here not to generate a TK_VECTOR containing only a - ** single value. Since the parser never creates such a vector, some - ** of the subroutines do not handle this case. */ - if( pLhs->nExpr==1 ){ - pX->pLeft = pLhs->a[0].pExpr; - }else{ - pLeft->x.pList = pLhs; - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq); - testcase( aiMap==0 ); - } - pSelect->pEList = pRhs; - db->dbOptFlags |= SQLITE_QueryFlattener; + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap); - db->dbOptFlags = savedDbOptFlags; - testcase( aiMap!=0 && aiMap[0]!=0 ); - pSelect->pEList = pOrigRhs; - pLeft->x.pList = pOrigLhs; - pX->pLeft = pLeft; + pTerm->pExpr->iTable = pX->iTable; } - sqlite3ExprListDelete(pParse->db, pLhs); - sqlite3ExprListDelete(pParse->db, pRhs); + sqlite3ExprDelete(db, pX); + pX = pTerm->pExpr; } if( eType==IN_INDEX_INDEX_DESC ){ From eebf2f5747182c8a46e4eb58fc9e8dc2567ed2cf Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 18 Nov 2017 17:30:08 +0000 Subject: [PATCH 139/156] Enhance the log messages produced in some cases if database corruption is encountered by an SQLITE_DEBUG build. FossilOrigin-Name: 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f --- manifest | 21 +++++++----- manifest.uuid | 2 +- src/btree.c | 91 +++++++++++++++++++++++++++++++------------------ src/main.c | 14 ++++---- src/sqliteInt.h | 1 + 5 files changed, 79 insertions(+), 50 deletions(-) diff --git a/manifest b/manifest index b8aae7e011..344babb1fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sfix\sfor\sticket\s[da78413751863]\sthat\sdoes\snot\srequire\sdisabling\sthe\nquery\sflattener\sas\swas\sdone\sin\s[005d5b870625].\s\sThis\salso\smakes\sthe\scode\ngenerator\sfor\svector\sIN\soperators\sa\slittle\seasier\sto\sunderstand. -D 2017-11-17T21:01:04.980 +C Enhance\sthe\slog\smessages\sproduced\sin\ssome\scases\sif\sdatabase\scorruption\sis\nencountered\sby\san\sSQLITE_DEBUG\sbuild. +D 2017-11-18T17:30:08.436 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -412,7 +412,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea +F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa @@ -435,7 +435,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5 +F src/main.c 99ed3d45e315afb2ada049991db7944b1210663bb30bfd0b63103537c1ac25d0 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -475,7 +475,7 @@ F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a90 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h fb297e4b891608057e857d583e30a261d905a3f41493f351fc91bae7d22008ff +F src/sqliteInt.h 9b26fbab75ef426efae70d88ab535844d59de8954542b122c1d49af580a76f58 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1677,7 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26 -R 04b5e48559de23ef7902e501e8dcaac8 -U drh -Z cc87a690f1d098ebb337ab16a1f38777 +P 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f +R 05158c3570d45bd02cc2ba4cbb719bc9 +T *branch * sqlite-corrupt-page +T *sym-sqlite-corrupt-page * +T -sym-trunk * +U dan +Z b072798007848ebbd313378cc8741324 diff --git a/manifest.uuid b/manifest.uuid index 99c0990a4c..dfb35df8dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f \ No newline at end of file +23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ddcb6cfd35..0fa00a2e12 100644 --- a/src/btree.c +++ b/src/btree.c @@ -112,6 +112,31 @@ int sqlite3_enable_shared_cache(int enable){ #define hasReadConflicts(a, b) 0 #endif +/* +** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single +** (MemPage*) as an argument. The (MemPage*) must not be NULL. +** +** If SQLITE_DEBUG is not defined, then this macro is equivalent to +** SQLITE_CORRUPT_BKPT. Or, if SQLITE_DEBUG is set, then the log message +** normally produced as a side-effect of SQLITE_CORRUPT_BKPT is augmented +** with the page number and filename associated with the (MemPage*). +*/ +#ifdef SQLITE_DEBUG +int corruptPageError(int lineno, MemPage *p){ + char *zMsg = sqlite3_mprintf("database corruption page %d of %s", + (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + ); + if( zMsg ){ + sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); + } + sqlite3_free(zMsg); + return SQLITE_CORRUPT_BKPT; +} +# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) +#else +# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) +#endif + #ifndef SQLITE_OMIT_SHARED_CACHE #ifdef SQLITE_DEBUG @@ -1400,7 +1425,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ int sz = get2byte(&data[iFree+2]); int top = get2byte(&data[hdr+5]); if( top>=iFree ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } if( iFree2 ){ assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */ @@ -1434,13 +1459,13 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** if PRAGMA cell_size_check=ON. */ if( pciCellLast ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } assert( pc>=iCellFirst && pc<=iCellLast ); size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; if( cbrkusableSize ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); testcase( cbrk+size==usableSize ); @@ -1460,7 +1485,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ defragment_out: if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -1504,7 +1529,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ testcase( x==4 ); testcase( x==3 ); if( size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno); + *pRc = SQLITE_CORRUPT_PAGE(pPg); return 0; }else if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total @@ -1527,7 +1552,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ if( pcpgno); + *pRc = SQLITE_CORRUPT_PAGE(pPg); } return 0; @@ -1575,7 +1600,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } } @@ -1665,12 +1690,12 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ while( (iFreeBlk = get2byte(&data[iPtr]))pgno); + return SQLITE_CORRUPT_PAGE(pPage); } iPtr = iFreeBlk; } if( iFreeBlk>pPage->pBt->usableSize-4 ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } assert( iFreeBlk>iPtr || iFreeBlk==0 ); @@ -1682,10 +1707,10 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ */ if( iFreeBlk && iEnd+3>=iFreeBlk ){ nFrag = iFreeBlk - iEnd; - if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); if( iEnd > pPage->pBt->usableSize ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); @@ -1698,13 +1723,13 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( iPtr>hdr+1 ){ int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); if( iPtrEnd+3>=iStart ){ - if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); nFrag += iStart - iPtrEnd; iSize = iEnd - iPtr; iStart = iPtr; } } - if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); data[hdr+7] -= nFrag; } x = get2byte(&data[hdr+5]); @@ -1712,7 +1737,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ - if( iStartpgno); + if( iStartpgno); + return SQLITE_CORRUPT_PAGE(pPage); } pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; @@ -1826,7 +1851,7 @@ static int btreeInitPage(MemPage *pPage){ /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -1845,7 +1870,7 @@ static int btreeInitPage(MemPage *pPage){ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } testcase( pPage->nCell==MX_CELL(pBt) ); /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only @@ -1873,12 +1898,12 @@ static int btreeInitPage(MemPage *pPage){ testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } } if( !pPage->leaf ) iCellLast++; @@ -1896,12 +1921,12 @@ static int btreeInitPage(MemPage *pPage){ /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will ** always be at least one cell before the first freeblock. */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } while( 1 ){ if( pc>iCellLast ){ /* Freeblock off the end of the page */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } next = get2byte(&data[pc]); size = get2byte(&data[pc+2]); @@ -1911,11 +1936,11 @@ static int btreeInitPage(MemPage *pPage){ } if( next>0 ){ /* Freeblock not in ascending order */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } if( pc+size>(unsigned int)usableSize ){ /* Last freeblock extends past page end */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } } @@ -1927,7 +1952,7 @@ static int btreeInitPage(MemPage *pPage){ ** area, according to the page header, lies within the page. */ if( nFree>usableSize ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } pPage->nFree = (u16)(nFree - iCellFirst); pPage->isInit = 1; @@ -3458,7 +3483,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW2 ){ /* The pointer is always the first 4 bytes of the page in this case. */ if( get4byte(pPage->aData)!=iFrom ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } put4byte(pPage->aData, iTo); }else{ @@ -3477,7 +3502,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ pPage->xParseCell(pPage, pCell, &info); if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } if( iFrom==get4byte(pCell+info.nSize-4) ){ put4byte(pCell+info.nSize-4, iTo); @@ -3495,7 +3520,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( i==nCell ){ if( eType!=PTRMAP_BTREE || get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); } @@ -4593,7 +4618,7 @@ static int accessPayload( ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ** but is recast into its current form to avoid integer overflow problems */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } /* Check if data must be read/written to/from the btree page itself. */ @@ -4741,7 +4766,7 @@ static int accessPayload( if( rc==SQLITE_OK && amt>0 ){ /* Overflow chain ends prematurely */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } return rc; } @@ -5019,7 +5044,7 @@ static int moveToRoot(BtCursor *pCur){ ** (or the freelist). */ assert( pRoot->intKey==1 || pRoot->intKey==0 ); if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ - return SQLITE_CORRUPT_PGNO(pCur->pPage->pgno); + return SQLITE_CORRUPT_PAGE(pCur->pPage); } skip_init: @@ -5292,7 +5317,7 @@ int sqlite3BtreeMovetoUnpacked( if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ){ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } } } @@ -5366,7 +5391,7 @@ int sqlite3BtreeMovetoUnpacked( testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ testcase( nCell==2 ); /* Minimum legal index key size */ if( nCell<2 ){ - rc = SQLITE_CORRUPT_PGNO(pPage->pgno); + rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_finish; } pCellKey = sqlite3Malloc( nCell+18 ); @@ -6169,7 +6194,7 @@ static int clearCell( } if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){ /* Cell extends past end of page */ - return SQLITE_CORRUPT_PGNO(pPage->pgno); + return SQLITE_CORRUPT_PAGE(pPage); } ovflPgno = get4byte(pCell + pInfo->nSize - 4); pBt = pPage->pBt; diff --git a/src/main.c b/src/main.c index d090845ae3..f9b34f80e8 100644 --- a/src/main.c +++ b/src/main.c @@ -3344,37 +3344,37 @@ int sqlite3_get_autocommit(sqlite3 *db){ ** 2. Invoke sqlite3_log() to provide the source code location where ** a low-level error is first detected. */ -static int reportError(int iErr, int lineno, const char *zType){ +int sqlite3ReportError(int iErr, int lineno, const char *zType){ sqlite3_log(iErr, "%s at line %d of [%.10s]", zType, lineno, 20+sqlite3_sourceid()); return iErr; } int sqlite3CorruptError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CORRUPT, lineno, "database corruption"); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); } int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_MISUSE, lineno, "misuse"); + return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); } int sqlite3CantopenError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); + return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); } #ifdef SQLITE_DEBUG int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ char zMsg[100]; sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CORRUPT, lineno, zMsg); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); } int sqlite3NomemError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_NOMEM, lineno, "OOM"); + return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); } int sqlite3IoerrnomemError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); + return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 058fee1461..c4e3a7dabc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3431,6 +3431,7 @@ struct TreeView { ** using sqlite3_log(). The routines also provide a convenient place ** to set a debugger breakpoint. */ +int sqlite3ReportError(int iErr, int lineno, const char *zType); int sqlite3CorruptError(int); int sqlite3MisuseError(int); int sqlite3CantopenError(int); From 75dbf68b5b8b91d209702b17db48b70af43c5f96 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Nov 2017 14:40:03 +0000 Subject: [PATCH 140/156] Fix a problem preventing the planner from identifying scans that visit at most one row in cases where that property is guaranteed by a unique, not-null, non-IPK column that is the leftmost in its table. FossilOrigin-Name: 299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- test/join2.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b35a1be8bc..12b73310c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\slog\smessages\sproduced\sin\ssome\scases\sif\sdatabase\scorruption\sis\nencountered\sby\san\sSQLITE_DEBUG\sbuild. -D 2017-11-18T18:07:17.406 +C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable. +D 2017-11-20T14:40:03.587 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db +F src/where.c e729f477523e3394892951347f57da1471a94a601133c19efd2573903e60a4a2 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test a48f723c5692e2cbb23a9297ac2720cb77d51a70 +F test/join2.test 8e3085b352e9f6d89247059ffd55c572b517f00295c8c0643a5c20cabd1e3848 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f -R 05158c3570d45bd02cc2ba4cbb719bc9 -T +closed 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f +P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3 +R d6874f2de8f8cac9d7779bbf432e0b3c U dan -Z ce33cab931a12d704bb3e13109bc56c2 +Z 3df96851db88f42b898e0f724b99ed03 diff --git a/manifest.uuid b/manifest.uuid index f4e48d0118..64fa4f8dcf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3 \ No newline at end of file +299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5545a45e87..955b9450ef 100644 --- a/src/where.c +++ b/src/where.c @@ -2460,7 +2460,7 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_EQ; assert( saved_nEq==pNew->u.btree.nEq ); if( iCol==XN_ROWID - || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; diff --git a/test/join2.test b/test/join2.test index 9372e770c3..850338955f 100644 --- a/test/join2.test +++ b/test/join2.test @@ -92,4 +92,35 @@ do_catchsql_test 2.2 { SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c); } {0 {one one one}} +#------------------------------------------------------------------------- +# Test that a problem causing where.c to overlook opportunities to +# omit unnecessary tables from a LEFT JOIN when UNIQUE, NOT NULL column +# that makes this possible happens to be the leftmost in its table. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(k1 INTEGER PRIMARY KEY, k2, k3); + CREATE TABLE t2(k2 INTEGER PRIMARY KEY, v2); + + # Prior to this problem being fixed, table t3_2 would be omitted from + # the join queries below, but if t3_1 were used in its place it would + # not. + CREATE TABLE t3_1(k3 PRIMARY KEY, v3) WITHOUT ROWID; + CREATE TABLE t3_2(v3, k3 PRIMARY KEY) WITHOUT ROWID; +} + +do_eqp_test 3.1 { + SELECT v2 FROM t1 LEFT JOIN t2 USING (k2) LEFT JOIN t3_1 USING (k3); +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} +} + +do_eqp_test 3.2 { + SELECT v2 FROM t1 LEFT JOIN t2 USING (k2) LEFT JOIN t3_2 USING (k3); +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} +} + finish_test From 17f188e3cfcba13a96bf0fddbe61e3c9d16e35fa Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Nov 2017 15:45:03 +0000 Subject: [PATCH 141/156] Fix a typo in a test script on this branch. FossilOrigin-Name: bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/join2.test | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 12b73310c0..b3ae26712d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable. -D 2017-11-20T14:40:03.587 +C Fix\sa\stypo\sin\sa\stest\sscript\son\sthis\sbranch. +D 2017-11-20T15:45:03.103 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test 8e3085b352e9f6d89247059ffd55c572b517f00295c8c0643a5c20cabd1e3848 +F test/join2.test 770a9b6c9f0e91590a102bae707727bce6103b8ad478cea077b2ac126d0698e8 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3 -R d6874f2de8f8cac9d7779bbf432e0b3c +P 299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d +R e3d55f8947932d938e221f1f7209d770 U dan -Z 3df96851db88f42b898e0f724b99ed03 +Z 5b68c810792c127ff29194bdbf236929 diff --git a/manifest.uuid b/manifest.uuid index 64fa4f8dcf..f9d78fd1e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d \ No newline at end of file +bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694 \ No newline at end of file diff --git a/test/join2.test b/test/join2.test index 850338955f..65a0ca266b 100644 --- a/test/join2.test +++ b/test/join2.test @@ -102,9 +102,9 @@ do_execsql_test 3.0 { CREATE TABLE t1(k1 INTEGER PRIMARY KEY, k2, k3); CREATE TABLE t2(k2 INTEGER PRIMARY KEY, v2); - # Prior to this problem being fixed, table t3_2 would be omitted from - # the join queries below, but if t3_1 were used in its place it would - # not. + -- Prior to this problem being fixed, table t3_2 would be omitted from + -- the join queries below, but if t3_1 were used in its place it would + -- not. CREATE TABLE t3_1(k3 PRIMARY KEY, v3) WITHOUT ROWID; CREATE TABLE t3_2(v3, k3 PRIMARY KEY) WITHOUT ROWID; } From 41203c6cf1e62bed4a39e72cb34e9128bf5108d0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Nov 2017 19:22:45 +0000 Subject: [PATCH 142/156] Update the omit-table-from-left-join optimization so that it can omit tables from the middle of the join as well as the end. FossilOrigin-Name: 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401 --- manifest | 18 ++++++++------ manifest.uuid | 2 +- src/where.c | 65 +++++++++++++++++++++++++++++++++++++++---------- test/join2.test | 48 ++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ace7d54144..cd6879f39b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable. -D 2017-11-20T15:46:10.484 +C Update\sthe\somit-table-from-left-join\soptimization\sso\sthat\sit\scan\somit\stables\nfrom\sthe\smiddle\sof\sthe\sjoin\sas\swell\sas\sthe\send. +D 2017-11-21T19:22:45.416 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c e729f477523e3394892951347f57da1471a94a601133c19efd2573903e60a4a2 +F src/where.c 9742731e325768332d67df217eb636556c8605f895d518ee61e4d13029f8aed8 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test 770a9b6c9f0e91590a102bae707727bce6103b8ad478cea077b2ac126d0698e8 +F test/join2.test ac70b2b79ac593550d1d6f15e9bb3693dd71826b496e84f15166c5053fa518e6 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1677,8 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3 bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694 -R e3d55f8947932d938e221f1f7209d770 -T +closed bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694 +P 7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7 +R 77239b01fe37a025cc689347a6c626c1 +T *branch * left-join-optimization +T *sym-left-join-optimization * +T -sym-trunk * U dan -Z 0d562170c07312a10559bcbda1122c53 +Z 55e1c0e690d530335519db505e442adf diff --git a/manifest.uuid b/manifest.uuid index 258ed7d3a3..ea81b36077 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7 \ No newline at end of file +618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 955b9450ef..0ced477da4 100644 --- a/src/where.c +++ b/src/where.c @@ -4677,35 +4677,73 @@ WhereInfo *sqlite3WhereBegin( } } #endif - /* Attempt to omit tables from the join that do not effect the result */ + + /* Attempt to omit tables from the join that do not affect the result. + ** For a table to not affect the result, the following must be true: + ** + ** 1) The query must not be an aggregate. + ** 2) The table must be the RHS of a LEFT JOIN. + ** 3) Either the query must be DISTINCT, or else the ON or USING clause + ** must contain a constraint that limits the scan of the table to + ** at most a single row. + ** 4) The table must not be referenced by any part of the query apart + ** from its own USING or ON clause. + ** + ** For example, given: + ** + ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); + ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); + ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); + ** + ** then table t2 can be omitted from the following: + ** + ** SELECT v1, v3 FROM t1 + ** LEFT JOIN t2 USING (t1.ipk=t2.ipk) + ** LEFT JOIN t3 USING (t1.ipk=t3.ipk) + ** + ** or from: + ** + ** SELECT DISTINCT v1, v3 FROM t1 + ** LEFT JOIN t2 + ** LEFT JOIN t3 USING (t1.ipk=t3.ipk) + */ if( pWInfo->nLevel>=2 - && pResultSet!=0 + && pResultSet!=0 /* guarantees condition (1) above */ && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ + int i; Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); if( sWLB.pOrderBy ){ tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); } - while( pWInfo->nLevel>=2 ){ + for(i=pWInfo->nLevel-1; i>=1; i--){ WhereTerm *pTerm, *pEnd; - pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; + struct SrcList_item *pItem; + pLoop = pWInfo->a[i].pWLoop; + pItem = &pWInfo->pTabList->a[pLoop->iTab]; + if( (pItem->fg.jointype & JT_LEFT)==0 ) continue; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ - break; + continue; } - if( (tabUsed & pLoop->maskSelf)!=0 ) break; + if( (tabUsed & pLoop->maskSelf)!=0 ) continue; pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - ){ - break; + if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ + if( !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + || pTerm->pExpr->iRightJoinTable!=pItem->iCursor + ){ + break; + } } } - if( pTerm drop loop %c not used\n", pLoop->cId)); + if( i!=pWInfo->nLevel-1 ){ + int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); + memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); + } pWInfo->nLevel--; nTabList--; } @@ -4990,7 +5028,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) diff --git a/test/join2.test b/test/join2.test index 65a0ca266b..9fdcef1783 100644 --- a/test/join2.test +++ b/test/join2.test @@ -123,4 +123,52 @@ do_eqp_test 3.2 { 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } +#------------------------------------------------------------------------- +# Test that tables other than the rightmost can be omitted from a +# LEFT JOIN query. +# +do_execsql_test 4.0 { + CREATE TABLE c1(k INTEGER PRIMARY KEY, v1); + CREATE TABLE c2(k INTEGER PRIMARY KEY, v2); + CREATE TABLE c3(k INTEGER PRIMARY KEY, v3); + + INSERT INTO c1 VALUES(1, 2); + INSERT INTO c2 VALUES(2, 3); + INSERT INTO c3 VALUES(3, 'v3'); + + INSERT INTO c1 VALUES(111, 1112); + INSERT INTO c2 VALUES(112, 1113); + INSERT INTO c3 VALUES(113, 'v1113'); +} +do_execsql_test 4.1.1 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} {2 v3 1112 {}} +do_execsql_test 4.1.2 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.1.3 { + SELECT DISTINCT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.1.4 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 2 v3 1112 {} 1112 {}} + +do_eqp_test 4.2.1 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 1 {SEARCH TABLE c2 USING INTEGER PRIMARY KEY (rowid=?)} + 0 2 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} +} +do_eqp_test 4.2.2 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} +} + + + finish_test From c6bc15a804eb5a37a1159d2ef7b0e8c12844e44e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 21 Nov 2017 21:14:32 +0000 Subject: [PATCH 143/156] Fix compilation issue (C99-ism) in the shell seen with MSVC. FossilOrigin-Name: 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/shell.c.in | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3cff6827c1..6b501fa59f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\somit-table-from-left-join\soptimization\sso\sthat\sit\scan\somit\stables\nfrom\sthe\smiddle\sof\sthe\sjoin\sas\swell\sas\sthe\send. -D 2017-11-21T20:53:14.147 +C Fix\scompilation\sissue\s(C99-ism)\sin\sthe\sshell\sseen\swith\sMSVC. +D 2017-11-21T21:14:32.355 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a909 +F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb8 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401 -R 77239b01fe37a025cc689347a6c626c1 -T +closed 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401 -U dan -Z 33c2b988566a650eaa7cbc98ff0c3943 +P 0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3 +R 4fa7a7c4266f9577164091bee69af1d6 +U mistachkin +Z 932ce0ecb18fdaa178b1c4aecbb344e5 diff --git a/manifest.uuid b/manifest.uuid index 2aded3bcbc..7486e65b91 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3 \ No newline at end of file +9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 68a337db11..f41e7438ed 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5952,8 +5952,10 @@ static int do_meta_command(char *zLine, ShellState *p){ int rc2 = 0; /* 0: usage. 1: %d 2: %x 3: no-output */ int isOk = 0; int i, n2; + const char *zCmd = 0; + open_db(p, 0); - const char *zCmd = nArg>=2 ? azArg[1] : "help"; + zCmd = nArg>=2 ? azArg[1] : "help"; /* The argument can optionally begin with "-" or "--" */ if( zCmd[0]=='-' && zCmd[1] ){ From 041343dea141b51974a465ed2fb3412edadd7bee Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 21 Nov 2017 21:15:17 +0000 Subject: [PATCH 144/156] Fix typo in the Makefiles for MSVC. FossilOrigin-Name: 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8 --- Makefile.msc | 4 ++-- autoconf/Makefile.msc | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 9c834931d7..b16f08adee 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -974,7 +974,7 @@ LTLINK = $(TCC) -Fe$@ # If requested, link to the RPCRT4 library. # !IF $(USE_RPCRT4_LIB)!=0 -LTLINK = $(LTLINK) rpcrt4.lib +LTLIBS = $(LTLIBS) rpcrt4.lib !ENDIF # If a platform was set, force the linker to target that. @@ -1072,7 +1072,7 @@ LDFLAGS = $(LDOPTS) # !IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) -LTLIBS = $(LIBTCL) +LTLIBS = $(LTLIBS) $(LIBTCL) !ENDIF # If ICU support is enabled, add the linker options for it. diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 39831da03e..199af21ea9 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -808,7 +808,7 @@ LTLINK = $(TCC) -Fe$@ # If requested, link to the RPCRT4 library. # !IF $(USE_RPCRT4_LIB)!=0 -LTLINK = $(LTLINK) rpcrt4.lib +LTLIBS = $(LTLIBS) rpcrt4.lib !ENDIF # If a platform was set, force the linker to target that. diff --git a/manifest b/manifest index 6b501fa59f..333b9dce22 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\scompilation\sissue\s(C99-ism)\sin\sthe\sshell\sseen\swith\sMSVC. -D 2017-11-21T21:14:32.355 +C Fix\stypo\sin\sthe\sMakefiles\sfor\sMSVC. +D 2017-11-21T21:15:17.379 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b +F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39 -F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac +F autoconf/Makefile.msc b88a70dee8453cc353e5d6df172d60a11a0af905710a24b1e6be80f8fea6e96b F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3 -R 4fa7a7c4266f9577164091bee69af1d6 +P 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252 +R a6bc597bdf966513d392563d9213ee42 U mistachkin -Z 932ce0ecb18fdaa178b1c4aecbb344e5 +Z c9f491fa6730dc4831f9db1bff90ef98 diff --git a/manifest.uuid b/manifest.uuid index 7486e65b91..33f6abfa72 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252 \ No newline at end of file +17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8 \ No newline at end of file From 065b34f6e54d420a6a3a73e1394cc4560b3d64aa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Nov 2017 23:38:48 +0000 Subject: [PATCH 145/156] Fix the skip-ahead-distinct optimization on joins for cases there the table in the inner loop of the join does not contribute any columns to the result set. Proposed fix for ticket [ef9318757b152e3a2] FossilOrigin-Name: 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 7 +++++++ test/distinct2.test | 19 +++++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 333b9dce22..a96176b69b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sthe\sMakefiles\sfor\sMSVC. -D 2017-11-21T21:15:17.379 +C Fix\sthe\sskip-ahead-distinct\soptimization\son\sjoins\sfor\scases\sthere\sthe\stable\nin\sthe\sinner\sloop\sof\sthe\sjoin\sdoes\snot\scontribute\sany\scolumns\sto\sthe\nresult\sset.\s\sProposed\sfix\sfor\sticket\s[ef9318757b152e3a2] +D 2017-11-21T23:38:48.349 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 9742731e325768332d67df217eb636556c8605f895d518ee61e4d13029f8aed8 +F src/where.c 1c0ab20720f1a5e76fb4f78d2c45b4525b59167a20d44baf4467fa5b106679ad F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -735,7 +735,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c -F test/distinct2.test faef8a3f27424e2cfbe19b2a40752294de3b5d957e049e3336be53ec0476cb58 +F test/distinct2.test ba239e9674495eda8e130cb5d498036b781dbd65c5868b58e094e36279859e83 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252 -R a6bc597bdf966513d392563d9213ee42 -U mistachkin -Z c9f491fa6730dc4831f9db1bff90ef98 +P 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8 +R c2dbc2aeaa2d6cfa4070ebdfc171fa89 +U drh +Z 8287b3f25cb0100d8a846549bf308df5 diff --git a/manifest.uuid b/manifest.uuid index 33f6abfa72..bc87796a6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8 \ No newline at end of file +2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0ced477da4..517d69fc1b 100644 --- a/src/where.c +++ b/src/where.c @@ -4978,6 +4978,13 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + if( inLevel-1 ){ + /* Ticket https://sqlite.org/src/info/ef9318757b152e3 2017-11-21 + ** The break location for the next inner loop is above the code + ** generated here, but it should be afterwards. So call re-resolve + ** the break location to be afterwards. */ + sqlite3VdbeResolveLabel(v, pWInfo->a[i+1].addrBrk); + } } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ /* The common case: Advance to the next row */ diff --git a/test/distinct2.test b/test/distinct2.test index a7d59db705..ae7a14cb1a 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -179,5 +179,24 @@ do_execsql_test 920 { wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz } +# Ticket https://sqlite.org/src/info/ef9318757b152e3a on 2017-11-21 +# Incorrect result due to a skip-ahead-distinct optimization on a +# join where no rows of the inner loop appear in the result set. +# +db close +sqlite3 db :memory: +do_execsql_test 1000 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER); + CREATE INDEX t1b ON t1(b); + CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER); + CREATE INDEX t2y ON t2(y); + WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<49) + INSERT INTO t1(b) SELECT x/10 - 1 FROM c; + WITH RECURSIVE c(x) AS (VALUES(-1) UNION ALL SELECT x+1 FROM c WHERE x<19) + INSERT INTO t2(x,y) SELECT x, 1 FROM c; + SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1; + ANALYZE; + SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1; +} {1 1} finish_test From fa337cc115c3151175607054607510f777a525d9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Nov 2017 00:45:21 +0000 Subject: [PATCH 146/156] Completely disable the skip-ahead-distinct optimization for all but the inner-most loop of a DISTINCT join. Fix for ticket [ef9318757b152e3a26e9592] FossilOrigin-Name: b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 8 +------- test/distinct2.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index a96176b69b..4044f32416 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sskip-ahead-distinct\soptimization\son\sjoins\sfor\scases\sthere\sthe\stable\nin\sthe\sinner\sloop\sof\sthe\sjoin\sdoes\snot\scontribute\sany\scolumns\sto\sthe\nresult\sset.\s\sProposed\sfix\sfor\sticket\s[ef9318757b152e3a2] -D 2017-11-21T23:38:48.349 +C Completely\sdisable\sthe\sskip-ahead-distinct\soptimization\sfor\sall\sbut\sthe\ninner-most\sloop\sof\sa\sDISTINCT\sjoin.\s\sFix\sfor\sticket\s[ef9318757b152e3a26e9592] +D 2017-11-23T00:45:21.475 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 1c0ab20720f1a5e76fb4f78d2c45b4525b59167a20d44baf4467fa5b106679ad +F src/where.c 1b85f0676c6b0e07bf815ce752e2f1e9968a4d506bfb1e9cb35dd95d18254fcf F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -735,7 +735,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c -F test/distinct2.test ba239e9674495eda8e130cb5d498036b781dbd65c5868b58e094e36279859e83 +F test/distinct2.test df0bb52b754661ea84ec9ff488d48913c97bd31d83ca17ce0bf1334645e660cf F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8 -R c2dbc2aeaa2d6cfa4070ebdfc171fa89 +P 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd +R 5a77744cb1beed3c66e0137b7a88aa75 U drh -Z 8287b3f25cb0100d8a846549bf308df5 +Z 8adc99178c09900ff045958fd53e1035 diff --git a/manifest.uuid b/manifest.uuid index bc87796a6f..0aaff2a8b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd \ No newline at end of file +b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 517d69fc1b..1a9f74dba0 100644 --- a/src/where.c +++ b/src/where.c @@ -4962,6 +4962,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ Index *pIdx; int n; if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ && (pLoop->wsFlags & WHERE_INDEXED)!=0 && (pIdx = pLoop->u.btree.pIndex)->hasStat1 && (n = pLoop->u.btree.nIdxCol)>0 @@ -4978,13 +4979,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); - if( inLevel-1 ){ - /* Ticket https://sqlite.org/src/info/ef9318757b152e3 2017-11-21 - ** The break location for the next inner loop is above the code - ** generated here, but it should be afterwards. So call re-resolve - ** the break location to be afterwards. */ - sqlite3VdbeResolveLabel(v, pWInfo->a[i+1].addrBrk); - } } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ /* The common case: Advance to the next row */ diff --git a/test/distinct2.test b/test/distinct2.test index ae7a14cb1a..31ab355132 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -198,5 +198,36 @@ do_execsql_test 1000 { ANALYZE; SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1; } {1 1} +db close +sqlite3 db :memory: +do_execsql_test 1010 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER); + CREATE INDEX t1b ON t1(b); + CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER); + CREATE INDEX t2y ON t2(y); + WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<49) + INSERT INTO t1(b) SELECT -(x/10 - 1) FROM c; + WITH RECURSIVE c(x) AS (VALUES(-1) UNION ALL SELECT x+1 FROM c WHERE x<19) + INSERT INTO t2(x,y) SELECT -x, 1 FROM c; + SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>1 ORDER BY y DESC; + ANALYZE; + SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>1 ORDER BY y DESC; +} {1 1} +db close +sqlite3 db :memory: +do_execsql_test 1020 { + CREATE TABLE t1(a, b); + CREATE INDEX t1a ON t1(a, b); + -- Lots of rows of (1, 'no'), followed by a single (1, 'yes'). + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a, b) SELECT 1, 'no' FROM c; + INSERT INTO t1(a, b) VALUES(1, 'yes'); + CREATE TABLE t2(x PRIMARY KEY); + INSERT INTO t2 VALUES('yes'); + SELECT DISTINCT a FROM t1, t2 WHERE x=b; + ANALYZE; + SELECT DISTINCT a FROM t1, t2 WHERE x=b; +} {1 1} + finish_test From 53bf7175afd5cc963f2f863c7ee62e160bd9f26d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Nov 2017 04:45:35 +0000 Subject: [PATCH 147/156] Fix a problem in the omit-table-from-left-join optimization from check-in [0cd82ee9a8413cf] that was discovered by OSSFuzz. FossilOrigin-Name: b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 8 +++++++- test/join2.test | 23 ++++++++++++++++++++++- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4044f32416..d89d6d2180 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Completely\sdisable\sthe\sskip-ahead-distinct\soptimization\sfor\sall\sbut\sthe\ninner-most\sloop\sof\sa\sDISTINCT\sjoin.\s\sFix\sfor\sticket\s[ef9318757b152e3a26e9592] -D 2017-11-23T00:45:21.475 +C Fix\sa\sproblem\sin\sthe\somit-table-from-left-join\soptimization\s\nfrom\scheck-in\s[0cd82ee9a8413cf]\sthat\swas\sdiscovered\sby\sOSSFuzz. +D 2017-11-23T04:45:35.235 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 1b85f0676c6b0e07bf815ce752e2f1e9968a4d506bfb1e9cb35dd95d18254fcf +F src/where.c 9752b68e03e2044f0faa4708fabb0189769067b660bffa931e1fd65736269659 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 @@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test ac70b2b79ac593550d1d6f15e9bb3693dd71826b496e84f15166c5053fa518e6 +F test/join2.test 1a0c26399910b015d9f8f95b884e9a079fd2cfdccd65f7b1603846508cae0dc6 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd -R 5a77744cb1beed3c66e0137b7a88aa75 +P b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d +R 9669b3f605458d14cc31b00c90e7e717 U drh -Z 8adc99178c09900ff045958fd53e1035 +Z 772d565646149360b17fb446b36f96b2 diff --git a/manifest.uuid b/manifest.uuid index 0aaff2a8b0..adb6476bf6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d \ No newline at end of file +b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1a9f74dba0..27979efbc4 100644 --- a/src/where.c +++ b/src/where.c @@ -4707,6 +4707,7 @@ WhereInfo *sqlite3WhereBegin( ** LEFT JOIN t2 ** LEFT JOIN t3 USING (t1.ipk=t3.ipk) */ + notReady = ~(Bitmask)0; if( pWInfo->nLevel>=2 && pResultSet!=0 /* guarantees condition (1) above */ && OptimizationEnabled(db, SQLITE_OmitNoopJoin) @@ -4740,6 +4741,12 @@ WhereInfo *sqlite3WhereBegin( } if( pTerm drop loop %c not used\n", pLoop->cId)); + notReady &= ~pLoop->maskSelf; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } if( i!=pWInfo->nLevel-1 ){ int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); @@ -4898,7 +4905,6 @@ WhereInfo *sqlite3WhereBegin( ** loop below generates code for a single nested loop of the VM ** program. */ - notReady = ~(Bitmask)0; for(ii=0; ii Date: Fri, 24 Nov 2017 16:55:48 +0000 Subject: [PATCH 148/156] Enhance the configure script to detect zLib. FossilOrigin-Name: e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 --- Makefile.in | 3 ++ configure | 103 +++++++++++++++++++++++++++++++++++++++++++------- configure.ac | 6 +++ manifest | 16 ++++---- manifest.uuid | 2 +- 5 files changed, 107 insertions(+), 23 deletions(-) diff --git a/Makefile.in b/Makefile.in index 567f31215f..4883c1ac6b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -89,6 +89,9 @@ TCC += $(OPT_FEATURE_FLAGS) # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". TCC += $(OPTS) +# Add in compile-time options for some libraries used by extensions +TCC += @HAVE_ZLIB@ + # Version numbers and release number for the SQLite being compiled. # VERSION = @VERSION@ diff --git a/configure b/configure index 9e7aa526a2..8e735c6b12 100755 --- a/configure +++ b/configure @@ -772,6 +772,7 @@ LIBOBJS BUILD_CFLAGS USE_GCOV OPT_FEATURE_FLAGS +HAVE_ZLIB USE_AMALGAMATION TARGET_DEBUG TARGET_HAVE_EDITLINE @@ -3931,13 +3932,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3934: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3935: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3938: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3940: output\"" >&5) + (eval echo "\"\$as_me:3941: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5143,7 +5144,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5146 "configure"' > conftest.$ac_ext + echo '#line 5147 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6668,11 +6669,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6671: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6672: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6675: \$? = $ac_status" >&5 + echo "$as_me:6676: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7007,11 +7008,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7010: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7011: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7014: \$? = $ac_status" >&5 + echo "$as_me:7015: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7112,11 +7113,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7115: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7116: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7119: \$? = $ac_status" >&5 + echo "$as_me:7120: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7167,11 +7168,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7170: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7171: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7174: \$? = $ac_status" >&5 + echo "$as_me:7175: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9547,7 +9548,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9550 "configure" +#line 9551 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9643,7 +9644,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9646 "configure" +#line 9647 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -11274,6 +11275,80 @@ if test "${use_amalgamation}" != "yes" ; then fi +######### +# Look for zlib. Only needed by extensions and by the sqlite3.exe shell +for ac_header in zlib.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" +if test "x$ac_cv_header_zlib_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_ZLIB_H 1 +_ACEOF + +fi + +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5 +$as_echo_n "checking for library containing deflate... " >&6; } +if ${ac_cv_search_deflate+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char deflate (); +int +main () +{ +return deflate (); + ; + return 0; +} +_ACEOF +for ac_lib in '' z; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_deflate=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_deflate+:} false; then : + break +fi +done +if ${ac_cv_search_deflate+:} false; then : + +else + ac_cv_search_deflate=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5 +$as_echo "$ac_cv_search_deflate" >&6; } +ac_res=$ac_cv_search_deflate +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1" +else + HAVE_ZLIB="" +fi + + + ######### # See whether we should allow loadable extensions # Check whether --enable-load-extension was given. diff --git a/configure.ac b/configure.ac index ace5d5d667..7089772d19 100644 --- a/configure.ac +++ b/configure.ac @@ -576,6 +576,12 @@ if test "${use_amalgamation}" != "yes" ; then fi AC_SUBST(USE_AMALGAMATION) +######### +# Look for zlib. Only needed by extensions and by the sqlite3.exe shell +AC_CHECK_HEADERS(zlib.h) +AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""]) +AC_SUBST(HAVE_ZLIB) + ######### # See whether we should allow loadable extensions AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension], diff --git a/manifest b/manifest index d89d6d2180..d56b60ea65 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Fix\sa\sproblem\sin\sthe\somit-table-from-left-join\soptimization\s\nfrom\scheck-in\s[0cd82ee9a8413cf]\sthat\swas\sdiscovered\sby\sOSSFuzz. -D 2017-11-23T04:45:35.235 -F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 +C Enhance\sthe\sconfigure\sscript\sto\sdetect\szLib. +D 2017-11-24T16:55:48.156 +F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd @@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure bdc49e9f0b0ced903ebdb2850362dd3391eeb88585e0429d12b94928d2873b6b x -F configure.ac 369ebae6c04d9d2de5064e21d300f2f42f2fbf13235cabff9d1a54f2b2c4d05d +F configure 9af547be0e0e1a8fca8553b82599b5a3be1528a3d78deb68cb49d3b611215cb7 x +F configure.ac d4529ebb26ae046269334f1dac65f2b1d6927c2efe22b2ec24dce24dfe4f83dd F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d -R 9669b3f605458d14cc31b00c90e7e717 +P b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa +R 32193c0299a7059451a8ff25bb895cce U drh -Z 772d565646149360b17fb446b36f96b2 +Z 88a3edc8b736c2bf1f1a3f732d1a86ca diff --git a/manifest.uuid b/manifest.uuid index adb6476bf6..bdb0f396fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa \ No newline at end of file +e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 \ No newline at end of file From d37ce8396a820cb01105d3c88b63e3922ff77c98 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Nov 2017 19:24:44 +0000 Subject: [PATCH 149/156] Add the "^" syntax from fts3/4 to fts5. FossilOrigin-Name: 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_expr.c | 62 +++++++++++++++------- ext/fts5/fts5parse.y | 7 ++- ext/fts5/test/fts5faultB.test | 17 +++++++ ext/fts5/test/fts5first.test | 96 +++++++++++++++++++++++++++++++++++ manifest | 21 ++++---- manifest.uuid | 2 +- 7 files changed, 176 insertions(+), 31 deletions(-) create mode 100644 ext/fts5/test/fts5first.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 63dc082687..c87583676b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -722,6 +722,8 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( int bPrefix ); +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*); + Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5Parse*, Fts5ExprNearset*, diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index aa7141cfee..a86dbebdf9 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -87,7 +87,8 @@ struct Fts5ExprNode { ** or term prefix. */ struct Fts5ExprTerm { - int bPrefix; /* True for a prefix term */ + u8 bPrefix; /* True for a prefix term */ + u8 bFirst; /* True if token must be first in column */ char *zTerm; /* nul-terminated term */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ @@ -168,6 +169,7 @@ static int fts5ExprGetToken( case '+': tok = FTS5_PLUS; break; case '*': tok = FTS5_STAR; break; case '-': tok = FTS5_MINUS; break; + case '^': tok = FTS5_CARET; break; case '\0': tok = FTS5_EOF; break; case '"': { @@ -427,6 +429,7 @@ static int fts5ExprPhraseIsMatch( Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; + int bFirst = pPhrase->aTerm[0].bFirst; fts5BufferZero(&pPhrase->poslist); @@ -481,8 +484,10 @@ static int fts5ExprPhraseIsMatch( }while( bMatch==0 ); /* Append position iPos to the output */ - rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); - if( rc!=SQLITE_OK ) goto ismatch_out; + if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; + } for(i=0; inTerm; i++){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; @@ -736,7 +741,9 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym + || pNear->pColset || pPhrase->aTerm[0].bFirst + ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); if( bMatch==0 ) break; @@ -917,6 +924,7 @@ static int fts5ExprNodeTest_STRING( assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 || pNear->apPhrase[0]->aTerm[0].pSynonym + || pNear->apPhrase[0]->aTerm[0].bFirst ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the @@ -1441,6 +1449,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ } } +/* +** Set the "bFirst" flag on the first token of the phrase passed as the +** only argument. +*/ +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){ + if( pPhrase && pPhrase->nTerm ){ + pPhrase->aTerm[0].bFirst = 1; + } +} + /* ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is @@ -1719,6 +1737,7 @@ int sqlite3Fts5ExprClonePhrase( } if( rc==SQLITE_OK ){ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; } } }else{ @@ -1737,7 +1756,10 @@ int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; - if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + if( pOrig->nTerm==1 + && pOrig->aTerm[0].pSynonym==0 + && pOrig->aTerm[0].bFirst==0 + ){ pNew->pRoot->eType = FTS5_TERM; pNew->pRoot->xNext = fts5ExprNodeNext_TERM; }else{ @@ -2011,6 +2033,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ Fts5ExprNearset *pNear = pNode->pNear; if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + && pNear->apPhrase[0]->aTerm[0].bFirst==0 ){ pNode->eType = FTS5_TERM; pNode->xNext = fts5ExprNodeNext_TERM; @@ -2097,20 +2120,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode( } } - if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL - && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) - ){ - assert( pParse->rc==SQLITE_OK ); - pParse->rc = SQLITE_ERROR; - assert( pParse->zErr==0 ); - pParse->zErr = sqlite3_mprintf( - "fts5: %s queries are not supported (detail!=full)", - pNear->nPhrase==1 ? "phrase": "NEAR" - ); - sqlite3_free(pRet); - pRet = 0; + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + if( pNear->nPhrase!=1 + || pPhrase->nTerm>1 + || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst) + ){ + assert( pParse->rc==SQLITE_OK ); + pParse->rc = SQLITE_ERROR; + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_mprintf( + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3_free(pRet); + pRet = 0; + } } - }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y index 1582909aa8..134125db1f 100644 --- a/ext/fts5/fts5parse.y +++ b/ext/fts5/fts5parse.y @@ -148,7 +148,11 @@ cnearset(A) ::= colset(X) COLON nearset(Y). { %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); } -nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); } +nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); } +nearset(A) ::= CARET phrase(Y). { + sqlite3Fts5ParseSetCaret(Y); + A = sqlite3Fts5ParseNearset(pParse, 0, Y); +} nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. { sqlite3Fts5ParseNear(pParse, &X); sqlite3Fts5ParseSetDistance(pParse, Y, &Z); @@ -189,6 +193,5 @@ phrase(A) ::= STRING(Y) star_opt(Z). { ** Optional "*" character. */ %type star_opt {int} - star_opt(A) ::= STAR. { A = 1; } star_opt(A) ::= . { A = 0; } diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test index a4fef523f5..2faec706d5 100644 --- a/ext/fts5/test/fts5faultB.test +++ b/ext/fts5/test/fts5faultB.test @@ -130,5 +130,22 @@ do_faultsim_test 4.2 -faults oom* -body { faultsim_test_result {0 {2 3}} } +#------------------------------------------------------------------------- +# Test OOM injection while parsing a CARET expression +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('a b c d'); -- 1 + INSERT INTO t1 VALUES('d a b c'); -- 2 + INSERT INTO t1 VALUES('c d a b'); -- 3 + INSERT INTO t1 VALUES('b c d a'); -- 4 +} +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT rowid FROM t1('^a OR ^b') } +} -test { + faultsim_test_result {0 {1 4}} +} + finish_test diff --git a/ext/fts5/test/fts5first.test b/ext/fts5/test/fts5first.test new file mode 100644 index 0000000000..b2cac1fdf1 --- /dev/null +++ b/ext/fts5/test/fts5first.test @@ -0,0 +1,96 @@ +# 2017 November 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5first + +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); +} + +foreach {tn expr ok} { + 1 {^abc} 1 + 2 {^abc + def} 1 + 3 {^ "abc def"} 1 + 4 {^"abc def"} 1 + 5 {abc ^def} 1 + 6 {abc + ^def} 0 + 7 {abc ^+ def} 0 + 8 {"^abc"} 1 + 9 {NEAR(^abc def)} 0 +} { + set res(0) {/1 {fts5: syntax error near .*}/} + set res(1) {0 {}} + + do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok) +} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + INSERT INTO x1 VALUES('a b c', 'b c a'); +} + +foreach {tn expr match} { + 1 {^a} 1 + 2 {^b} 1 + 3 {^c} 0 + 4 {^a + b} 1 + 5 {^b + c} 1 + 6 {^c + a} 0 + 7 {^"c a"} 0 + 8 {a:^a} 1 + 9 {a:^b} 0 + 10 {a:^"a b"} 1 +} { + do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + DELETE FROM x1; + INSERT INTO x1 VALUES('b a', 'c a'); + INSERT INTO x1 VALUES('a a', 'c c'); + INSERT INTO x1 VALUES('a b', 'a a'); +} +fts5_aux_test_functions db + +foreach {tn expr expect} { + 1 {^a} {{2 1}} + 2 {^c AND ^b} {{0 2} {1 0}} +} { + do_execsql_test 3.$tn { + SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1 + } [list $expect] +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column); +} + +do_catchsql_test 3.2 { + SELECT * FROM x2('a + b'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} + +do_catchsql_test 3.3 { + SELECT * FROM x2('^a'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} +finish_test + diff --git a/manifest b/manifest index d56b60ea65..fa07b8e660 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sconfigure\sscript\sto\sdetect\szLib. -D 2017-11-24T16:55:48.156 +C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5. +D 2017-11-24T19:24:44.918 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -99,11 +99,11 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 -F ext/fts5/fts5Int.h 15e7514b46a845937d7c62e5c69e935091f0dbb72eb61aa4c8bcfbd39fdea158 +F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c f2825f714d91bbe62ab5820aee9ad12e0c94205b2a01725eaa9072415ae9ff1c +F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55 F ext/fts5/fts5_index.c 2ce9d50ec5508b8205615aad69e1c9b2c77f017f21d4479e1fb2079c01fdd017 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b @@ -115,7 +115,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed0e7a77a6 -F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca +F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9 @@ -162,8 +162,9 @@ F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd1640 F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2 F ext/fts5/test/fts5fault9.test 0111b229388bdf251b91cfead68580227801dd30960a19aa8fe9021a1e73cb6d F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c53ca85af12290c8c -F ext/fts5/test/fts5faultB.test 28810d93d37b59ebd5cf9502897f4dc9e6adb8ea6a5f64e125d3088597199d0d +F ext/fts5/test/fts5faultB.test e6d04f9ea7b21be1d89abb8df2cb4baf65b0453b744d5a805fcd3ef45ff86a7e F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a +F ext/fts5/test/fts5first.test 707a591b1b7d893fcfcb2366cbfe56aefab5d9c7cfa58bef35eba73a1dbf3b29 F ext/fts5/test/fts5full.test 49b565da02918c06e58f51f0b953b0302b96f155aa68baba24782b81570685e2 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5hash.test a4cf51acad99bfc43c16fb74f9d22495dc221ae0701fc5e908ca963a9b26a02b @@ -1677,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa -R 32193c0299a7059451a8ff25bb895cce -U drh -Z 88a3edc8b736c2bf1f1a3f732d1a86ca +P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 +R dd0642a6cc4628ee924c556518e62685 +U dan +Z b13dcd2f6b4a5ff1701c0fa74c45ed53 diff --git a/manifest.uuid b/manifest.uuid index bdb0f396fc..45ca11ffc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 \ No newline at end of file +24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 \ No newline at end of file From 8385becfe153283312468feb5bf32219e6973940 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Nov 2017 17:51:01 +0000 Subject: [PATCH 150/156] Add experimental feature to detect threading bugs in apps that use SQLITE_CONFIG_MULTITHREADED. Enabled at compile time using SQLITE_ENABLE_MULTITHREADED_CHECKS. FossilOrigin-Name: a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d --- manifest | 19 ++--- manifest.uuid | 2 +- src/main.c | 10 ++- src/mutex.c | 181 ++++++++++++++++++++++++++++++++++++++++++++++++ src/sqliteInt.h | 6 ++ 5 files changed, 208 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index fa07b8e660..a55dbc7aa2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5. -D 2017-11-24T19:24:44.918 +C Add\sexperimental\sfeature\sto\sdetect\sthreading\sbugs\sin\sapps\sthat\suse\nSQLITE_CONFIG_MULTITHREADED.\sEnabled\sat\scompile\stime\susing\nSQLITE_ENABLE_MULTITHREADED_CHECKS. +D 2017-11-25T17:51:01.072 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -436,7 +436,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c 99ed3d45e315afb2ada049991db7944b1210663bb30bfd0b63103537c1ac25d0 +F src/main.c 6a0cc1c7b8ab92374effecdd7b92792b3273a255c70575b7d67bd9a4315e6d3a F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -445,7 +445,7 @@ F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 -F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c +F src/mutex.c 38addb10f90641b5f88521f7099e729ef10e3f0ac50bd6b9196183fb313a0378 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 @@ -476,7 +476,7 @@ F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h 9b26fbab75ef426efae70d88ab535844d59de8954542b122c1d49af580a76f58 +F src/sqliteInt.h 6b084f4b7c0ea0cde24f4400a88fca7108f1a304cb567338495bba46b0fd68c6 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1678,7 +1678,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 -R dd0642a6cc4628ee924c556518e62685 +P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 +R 1e88d93b9907cb6e6661756c6df7c284 +T *branch * multithreaded-checks +T *sym-multithreaded-checks * +T -sym-trunk * U dan -Z b13dcd2f6b4a5ff1701c0fa74c45ed53 +Z 7380701e6ba5cd3f46b8984142176402 diff --git a/manifest.uuid b/manifest.uuid index 45ca11ffc8..99d6d7ea4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 \ No newline at end of file +a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d \ No newline at end of file diff --git a/src/main.c b/src/main.c index f9b34f80e8..a7b631270e 100644 --- a/src/main.c +++ b/src/main.c @@ -2822,6 +2822,7 @@ static int openDatabase( }else{ isThreadsafe = sqlite3GlobalConfig.bFullMutex; } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ flags &= ~SQLITE_OPEN_SHAREDCACHE; }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ @@ -2854,13 +2855,20 @@ static int openDatabase( /* Allocate the sqlite data structure */ db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; - if( isThreadsafe ){ + if( isThreadsafe +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + || sqlite3GlobalConfig.bCoreMutex +#endif + ){ db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); if( db->mutex==0 ){ sqlite3_free(db); db = 0; goto opendb_out; } + if( isThreadsafe==0 ){ + sqlite3MutexWarnOnContention(db->mutex); + } } sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; diff --git a/src/mutex.c b/src/mutex.c index 6f1bc9767d..dd24e9be8c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -26,6 +26,182 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT + +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +/* +** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** the implementation of a wrapper around the system default mutex +** implementation (sqlite3DefaultMutex()). +** +** Most calls are passed directly through to the underlying default +** mutex implementation. Except, if a mutex is configured by calling +** sqlite3MutexWarnOnContention() on it, then if contention is ever +** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** +** This type of mutex is used as the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +*/ + +/* +** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** is defined. +*/ +typedef struct CheckMutex CheckMutex; +struct CheckMutex { + int iType; + sqlite3_mutex *mutex; +}; + +/* +** Pointer to real mutex methods object used by the CheckMutex +** implementation. Set by checkMutexInit(). +*/ +static SQLITE_WSD const sqlite3_mutex_methods *pGlobalMutexMethods; + +#ifdef SQLITE_DEBUG +static int checkMutexHeld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexHeld(((CheckMutex*)p)->mutex); +} +static int checkMutexNotheld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexNotheld(((CheckMutex*)p)->mutex); +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int checkMutexInit(void){ + pGlobalMutexMethods = sqlite3DefaultMutex(); + return SQLITE_OK; +} +static int checkMutexEnd(void){ + pGlobalMutexMethods = 0; + return SQLITE_OK; +} + +/* +** Allocate a mutex. +*/ +static sqlite3_mutex *checkMutexAlloc(int iType){ + static CheckMutex staticMutexes[] = { + {2, 0}, {3, 0}, {4, 0}, {5, 0}, + {6, 0}, {7, 0}, {8, 0}, {9, 0}, + {10, 0}, {11, 0}, {12, 0}, {13, 0} + }; + CheckMutex *p = 0; + + assert( SQLITE_MUTEX_RECURSIVE==1 && SQLITE_MUTEX_FAST==0 ); + if( iType<2 ){ + p = sqlite3MallocZero(sizeof(CheckMutex)); + if( p==0 ) return 0; + p->iType = iType; + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + } + + if( p->mutex==0 ){ + p->mutex = pGlobalMutexMethods->xMutexAlloc(iType); + if( p->mutex==0 ){ + if( iType<2 ){ + sqlite3_free(p); + } + p = 0; + } + } + + return (sqlite3_mutex*)p; +} + +/* +** Free a mutex. +*/ +static void checkMutexFree(sqlite3_mutex *p){ +#if SQLITE_ENABLE_API_ARMOR + if( p->iType<2 ){ +#endif + { + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexFree(pCheck->mutex); + sqlite3_free(pCheck); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** Enter the mutex. +*/ +static void checkMutexEnter(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + if( pCheck->iType<0 ){ + if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){ + return; + } + sqlite3_log(SQLITE_MISUSE, + "illegal multi-threaded access to database connection" + ); + } + pGlobalMutexMethods->xMutexEnter(pCheck->mutex); +} + +/* +** Enter the mutex (do not block). +*/ +static int checkMutexTry(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + return pGlobalMutexMethods->xMutexTry(pCheck->mutex); +} + +/* +** Leave the mutex. +*/ +static void checkMutexLeave(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexLeave(pCheck->mutex); +} + +sqlite3_mutex_methods const *multiThreadedCheckMutex(void){ + static const sqlite3_mutex_methods sMutex = { + checkMutexInit, + checkMutexEnd, + checkMutexAlloc, + checkMutexFree, + checkMutexEnter, + checkMutexTry, + checkMutexLeave, +#ifdef SQLITE_DEBUG + checkMutexHeld, + checkMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +/* +** Mark the SQLITE_MUTEX_RECURSIVE mutex passed as the only argument as +** one on which there should be no contention. +*/ +void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ + if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){ + CheckMutex *pCheck = (CheckMutex*)p; + assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE ); + pCheck->iType = -1; + } +} +#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ + /* ** Initialize the mutex system. */ @@ -41,7 +217,11 @@ int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + pFrom = multiThreadedCheckMutex(); +#else pFrom = sqlite3DefaultMutex(); +#endif }else{ pFrom = sqlite3NoopMutex(); } @@ -167,3 +347,4 @@ int sqlite3_mutex_notheld(sqlite3_mutex *p){ #endif #endif /* !defined(SQLITE_MUTEX_OMIT) */ + diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c4e3a7dabc..c416754eed 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3590,6 +3590,12 @@ int sqlite3LookasideUsed(sqlite3*,int*); sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +void sqlite3MutexWarnOnContention(sqlite3_mutex*); +#else +# define sqlite3MutexWarnOnContention(x) +#endif + #ifndef SQLITE_OMIT_FLOATING_POINT int sqlite3IsNaN(double); #else From 61f8e86f93ae0410bb93b27dfad4368539e5d67c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Nov 2017 21:09:29 +0000 Subject: [PATCH 151/156] Fix builds with both SQLITE_ENABLE_MULTITHREADED_CHECKS and SQLITE_THREADSAFE=0 defined. FossilOrigin-Name: 7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a55dbc7aa2..48f178e1ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\sfeature\sto\sdetect\sthreading\sbugs\sin\sapps\sthat\suse\nSQLITE_CONFIG_MULTITHREADED.\sEnabled\sat\scompile\stime\susing\nSQLITE_ENABLE_MULTITHREADED_CHECKS. -D 2017-11-25T17:51:01.072 +C Fix\sbuilds\swith\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_THREADSAFE=0\sdefined. +D 2017-11-25T21:09:29.272 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -476,7 +476,7 @@ F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h 6b084f4b7c0ea0cde24f4400a88fca7108f1a304cb567338495bba46b0fd68c6 +F src/sqliteInt.h 4c910d9c0d88a90e8639a4f83ef05f701ccfe731cf593e757444074f01df4964 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1678,10 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 -R 1e88d93b9907cb6e6661756c6df7c284 -T *branch * multithreaded-checks -T *sym-multithreaded-checks * -T -sym-trunk * +P a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d +R e1844ccd94c28116d72c961404a2fb64 U dan -Z 7380701e6ba5cd3f46b8984142176402 +Z 03bc6023c96e908c306e6b560dfce5a2 diff --git a/manifest.uuid b/manifest.uuid index 99d6d7ea4c..ec64aba6fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d \ No newline at end of file +7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c416754eed..27b896d67c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3590,7 +3590,7 @@ int sqlite3LookasideUsed(sqlite3*,int*); sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else # define sqlite3MutexWarnOnContention(x) From 9b7affc451d41afd060dd98b7b76605e92787142 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 26 Nov 2017 02:14:18 +0000 Subject: [PATCH 152/156] Fix an CSV output quoting problem in the command-line shell on Windows. FossilOrigin-Name: 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 9 +++------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index fa07b8e660..d1e9d5f842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5. -D 2017-11-24T19:24:44.918 +C Fix\san\sCSV\soutput\squoting\sproblem\sin\sthe\scommand-line\sshell\son\sWindows. +D 2017-11-26T02:14:18.907 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -472,7 +472,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb8 +F src/shell.c.in c441d7ddfbb8120cd8a7cde838ca5c9167311a7e400b1077c3ae6090aa420be8 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115 -R dd0642a6cc4628ee924c556518e62685 -U dan -Z b13dcd2f6b4a5ff1701c0fa74c45ed53 +P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 +R 8e5988afe15fc17639c16180584a05a2 +U drh +Z 78b26e27b3171f1fcd46be47e5730c56 diff --git a/manifest.uuid b/manifest.uuid index 45ca11ffc8..9e0d368055 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 \ No newline at end of file +6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f41e7438ed..daa887ef5d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1190,12 +1190,9 @@ static void output_csv(ShellState *p, const char *z, int bSep){ } } if( i==0 ){ - putc('"', out); - for(i=0; z[i]; i++){ - if( z[i]=='"' ) putc('"', out); - putc(z[i], out); - } - putc('"', out); + char *zQuoted = sqlite3_mprintf("\"%w\"", z); + utf8_printf(out, "%s", zQuoted); + sqlite3_free(zQuoted); }else{ utf8_printf(out, "%s", z); } From 7ac2d48eba5973fea8998ad80523ea5537b2b53f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Nov 2017 17:56:14 +0000 Subject: [PATCH 153/156] Fix a faulty NEVER assert() that could fail for SQLITE_ENABLE_STAT4 builds that use foreign keys. FossilOrigin-Name: 465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 2 +- test/fkey7.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d1e9d5f842..24f6855872 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sCSV\soutput\squoting\sproblem\sin\sthe\scommand-line\sshell\son\sWindows. -D 2017-11-26T02:14:18.907 +C Fix\sa\sfaulty\sNEVER\sassert()\sthat\scould\sfail\sfor\sSQLITE_ENABLE_STAT4\sbuilds\nthat\suse\sforeign\skeys. +D 2017-11-27T17:56:14.310 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -546,7 +546,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1 F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30 +F src/vdbemem.c 6e6082377183b8fd8d6f2bce45250b4c2e89bfeb534a094cd8612dae8aaa2fda F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a @@ -787,7 +787,7 @@ F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 24dd28eb3d9f1b5a174f47e9899ace5facb08373a4223593c8c631e6cf9f7d5a F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0 -F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 +F test/fkey7.test 24076d43d3449f12f25503909ca4bfb5fc5fefd5af1f930723a496343eb28454 F test/fkey8.test e5372e32cdb4481f121ec3550703eeb7b4e0762c F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/fordelete.test eb93a2f34137bb87bdab88fcab06c0bd92719aff @@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605 -R 8e5988afe15fc17639c16180584a05a2 -U drh -Z 78b26e27b3171f1fcd46be47e5730c56 +P 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589 +R 0247a4bb86c66f994a19293f6b9d8594 +U dan +Z a556263a76630f97a74cefb44456d7c1 diff --git a/manifest.uuid b/manifest.uuid index 9e0d368055..51e3848821 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589 \ No newline at end of file +465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index f9f58c43f4..3ec7394a45 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1321,7 +1321,7 @@ static int valueFromExpr( assert( pExpr!=0 ); while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; - if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + if( op==TK_REGISTER ) op = pExpr->op2; /* Compressed expressions only appear when parsing the DEFAULT clause ** on a table column definition, and hence only when pCtx==0. This diff --git a/test/fkey7.test b/test/fkey7.test index 6c646a9a7f..e86fc5c57b 100644 --- a/test/fkey7.test +++ b/test/fkey7.test @@ -68,4 +68,18 @@ ifcapable incrblob { } {SQLITE_CONSTRAINT} } +ifcapable stat4 { + do_execsql_test 3.0 { + CREATE TABLE p4 (id INTEGER NOT NULL PRIMARY KEY); + INSERT INTO p4 VALUES(1), (2), (3); + + CREATE TABLE c4(x INTEGER REFERENCES p4(id) DEFERRABLE INITIALLY DEFERRED); + CREATE INDEX c4_x ON c4(x); + INSERT INTO c4 VALUES(1), (2), (3); + + ANALYZE; + INSERT INTO p4(id) VALUES(4); + } +} + finish_test From cb45eef4d1de63685812dbbad1fc1d1dc1b76401 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Nov 2017 00:52:14 +0000 Subject: [PATCH 154/156] Update tests to deal with SQLITE_FAST_SECURE_DELETE. FossilOrigin-Name: e6b89304695be371978e65dddd710c8bd563c66b9c94d23165142b6c235c82e1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_config.c | 6 ++++++ test/securedel.test | 8 ++++++-- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 24f6855872..9d698eeef1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sNEVER\sassert()\sthat\scould\sfail\sfor\sSQLITE_ENABLE_STAT4\sbuilds\nthat\suse\sforeign\skeys. -D 2017-11-27T17:56:14.310 +C Update\stests\sto\sdeal\swith\sSQLITE_FAST_SECURE_DELETE. +D 2017-11-28T00:52:14.148 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -496,7 +496,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c 3000f00b9b47b149d842059904c3fcab5f3871fb6aee7d7cc5756f0c64779ae3 +F src/test_config.c 2dad654eb81e90160f764f485bf7248e87b6d251e484c959de1aa04935acae8e F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 @@ -1173,7 +1173,7 @@ F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce1f9b -F test/securedel.test 5f997cb6bd38727b81e0985f53ec386c99db6441b2b9e6357240649d29017239 +F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 @@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589 -R 0247a4bb86c66f994a19293f6b9d8594 -U dan -Z a556263a76630f97a74cefb44456d7c1 +P 465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6 +R 5c3698b136ad7029a7e22749e9f1a061 +U drh +Z 455c99a49ae132932748abfe00c111c4 diff --git a/manifest.uuid b/manifest.uuid index 51e3848821..289fbf5515 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6 \ No newline at end of file +e6b89304695be371978e65dddd710c8bd563c66b9c94d23165142b6c235c82e1 \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 193f95edf9..8756876d19 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -696,6 +696,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "unlock_notify", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_FAST_SECURE_DELETE + Tcl_SetVar2(interp, "sqlite_options", "fast_secure_delete", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fast_secure_delete", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_SECURE_DELETE Tcl_SetVar2(interp, "sqlite_options", "secure_delete", "1", TCL_GLOBAL_ONLY); #else diff --git a/test/securedel.test b/test/securedel.test index a78f466031..8323a30497 100644 --- a/test/securedel.test +++ b/test/securedel.test @@ -17,8 +17,12 @@ source $testdir/tester.tcl unset -nocomplain DEFAULT_SECDEL set DEFAULT_SECDEL 0 -ifcapable secure_delete { - set DEFAULT_SECDEL 1 +ifcapable fast_secure_delete { + set DEFAULT_SECDEL 2 +} else { + ifcapable secure_delete { + set DEFAULT_SECDEL 1 + } } From a90a2d58a450538fee710f6958c2a19fdbfb033a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Nov 2017 07:47:57 +0000 Subject: [PATCH 155/156] Use a symbolic constant instead of a literal (-1) to identify a warn-on-contention mutex. FossilOrigin-Name: 12a23c0a66fac5c9674120b390f6abaeaba3f7ff04693b281af1eefb93d6f47c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mutex.c | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 48f178e1ea..6f5326f179 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbuilds\swith\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_THREADSAFE=0\sdefined. -D 2017-11-25T21:09:29.272 +C Use\sa\ssymbolic\sconstant\sinstead\sof\sa\sliteral\s(-1)\sto\sidentify\sa\nwarn-on-contention\smutex. +D 2017-11-28T07:47:57.520 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -445,7 +445,7 @@ F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 -F src/mutex.c 38addb10f90641b5f88521f7099e729ef10e3f0ac50bd6b9196183fb313a0378 +F src/mutex.c 20172f2cc43c4542f7860ab6bafdd16965822c4e56650d62628059a48c7f43c4 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 @@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d -R e1844ccd94c28116d72c961404a2fb64 +P 7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916 +R a940a7b7e2663c64f019728def2f503f U dan -Z 03bc6023c96e908c306e6b560dfce5a2 +Z d7c77380d4461a6101913d9af19a0151 diff --git a/manifest.uuid b/manifest.uuid index ec64aba6fd..60dae67939 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916 \ No newline at end of file +12a23c0a66fac5c9674120b390f6abaeaba3f7ff04693b281af1eefb93d6f47c \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index dd24e9be8c..9e1bb65553 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -44,7 +44,12 @@ static SQLITE_WSD int mutexIsInit = 0; /* ** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS -** is defined. +** is defined. Variable CheckMutex.mutex is a pointer to the real mutex +** allocated by the system mutex implementation. Variable iType is usually set +** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST +** or one of the static mutex identifiers. Or, if this is a recursive mutex +** that has been configured using sqlite3MutexWarnOnContention(), it is +** set to SQLITE_MUTEX_WARNONCONTENTION. */ typedef struct CheckMutex CheckMutex; struct CheckMutex { @@ -52,6 +57,8 @@ struct CheckMutex { sqlite3_mutex *mutex; }; +#define SQLITE_MUTEX_WARNONCONTENTION (-1) + /* ** Pointer to real mutex methods object used by the CheckMutex ** implementation. Set by checkMutexInit(). @@ -122,6 +129,10 @@ static sqlite3_mutex *checkMutexAlloc(int iType){ ** Free a mutex. */ static void checkMutexFree(sqlite3_mutex *p){ + assert( SQLITE_MUTEX_RECURSIVE<2 ); + assert( SQLITE_MUTEX_FAST<2 ); + assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); + #if SQLITE_ENABLE_API_ARMOR if( p->iType<2 ){ #endif @@ -142,7 +153,7 @@ static void checkMutexFree(sqlite3_mutex *p){ */ static void checkMutexEnter(sqlite3_mutex *p){ CheckMutex *pCheck = (CheckMutex*)p; - if( pCheck->iType<0 ){ + if( pCheck->iType==SQLITE_MUTEX_WARNONCONTENTION ){ if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){ return; } @@ -197,7 +208,7 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){ CheckMutex *pCheck = (CheckMutex*)p; assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE ); - pCheck->iType = -1; + pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; } } #endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ From 0895117512bd461ac97f432f20a2dbd91095e1d5 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Nov 2017 20:43:40 +0000 Subject: [PATCH 156/156] Fix an assertion fault found by OSSFuzz. FossilOrigin-Name: 75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/triggerG.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0d6ebb2ca6..3a1692e2ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Lock\sthe\swal\sfile\sfor\sall\ssnapshot\stransactions,\seven\sif\sthey\swould\snot\notherwise\srequire\sthis,\spreventing\scheckpointers\sand\swriters\sfrom\swrapping\sthe\nwal\sfile.\sThis\smeans\sthat\sif\sone\sconnection\shas\san\sopen\ssnapshot\stransaction\nit\sis\sguaranteed\sthat\sa\ssecond\sconnection\scan\sopen\sa\stransaction\son\sthe\ssame\nsnapshot. -D 2017-11-28T13:39:41.135 +C Fix\san\sassertion\sfault\sfound\sby\sOSSFuzz. +D 2017-11-28T20:43:40.167 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44 @@ -535,7 +535,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5 F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd -F src/trigger.c fc6be2a6e103d9e38b161e07d7db0ffb1f2218bd2f27ccdc0a3d1cc89e9cea0f +F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 @@ -1457,7 +1457,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f F test/triggerF.test 6a8c22bd058cf467f0c7d112afe87f7a8c579c0c4681b914b8f19020f48528a4 -F test/triggerG.test 175cafdc6399d85231a09e82e051b0e45a2fd1f23dd08ae715bc359716149ab6 +F test/triggerG.test d5caeef6144ede2426dd13211fd72248241ff2ebc68e12a4c0bf30f5faa21499 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 @@ -1679,8 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40b598c8392f030f6ed8c63ce81cb0426bb3984397c19c756215f6a569a40164 d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1 -R e37ca38ceb47340e2d23476ad044f736 -T +closed d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1 +P b81a31495bd27c1d96f7df653da3502054240cb5acf66b860da7f0f9b422a524 +R 419b1183c51a17454670ae5c0f948f4f U dan -Z a9d291c094a58941f666434dce6709fb +Z af229f4b2717b4f314275fdbf907e896 diff --git a/manifest.uuid b/manifest.uuid index 1c080520c0..3f2f6eee60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b81a31495bd27c1d96f7df653da3502054240cb5acf66b860da7f0f9b422a524 \ No newline at end of file +75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04 \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index a64fb291a0..d8aac2cc22 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -875,7 +875,7 @@ static TriggerPrg *codeRowTrigger( VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); transferParseError(pParse, pSubParse); - if( db->mallocFailed==0 ){ + if( db->mallocFailed==0 && pParse->nErr==0 ){ pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); } pProgram->nMem = pSubParse->nMem; diff --git a/test/triggerG.test b/test/triggerG.test index c770b5bd76..f5965e1707 100644 --- a/test/triggerG.test +++ b/test/triggerG.test @@ -62,4 +62,17 @@ do_execsql_test 200 { SELECT b FROM t2 ORDER BY b; } {20202 20203 20302 20303 30202 30203 30302 30303 40202 40203 40302 40303 50202 50203 50302 50303} +# At one point the following was causing an assert() to fail. +# +do_execsql_test 300 { + CREATE TABLE t4(x); + CREATE TRIGGER tr4 AFTER INSERT ON t4 BEGIN + SELECT 0x2147483648e0e0099 AS y WHERE y; + END; +} + +do_catchsql_test 310 { + INSERT INTO t4 VALUES(1); +} {1 {hex literal too big: 0x2147483648e0e0099}} + finish_test