From 8abc80b2945b7133eee896092643d259410c048c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 12 Aug 2017 01:09:06 +0000
Subject: [PATCH 001/270] Size and performance micro-optimization on
 sqlite3SrcListIndexedBy().

FossilOrigin-Name: 28a5aec118f9d078c9e7225f85cd48a91920b13908c540771e309557c3f15f84
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/build.c   |  8 +++++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index e3348517c5..10e55ba5d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarnings\sthat\sarise\sif\sthe\sPAGERTRACE\smacro\sis\sturned\son.\nThis\schanges\sdoes\snot\saffect\sproduction\sbuilds.
-D 2017-08-11T18:59:00.035
+C Size\sand\sperformance\smicro-optimization\son\ssqlite3SrcListIndexedBy().
+D 2017-08-12T01:09:06.103
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -399,7 +399,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 1a17ba1a765d80c3ca39ce33ff55f92e1f51eb84bbbdab5377f11d36b1515fa1
 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
 F src/btreeInt.h 97700795edf8a43245720414798b7b29d8e465aef46bf301ffacd431910c0da1
-F src/build.c 33b0f6055bd990ed052b96e71368acefcd98daa21ccf21f91aa90e8b769c2219
+F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
 F src/callback.c 930648a084a3adc741c6471adfbdc50ba47ba3542421cb80a26f259f467de65e
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -1646,7 +1646,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P aa49926dbffaae4f7c486be72ad814f381cca65c549f9d2605f47540a5f4be84
-R d8063074df6e7dea7a8298073c2e62a5
+P 831156a4bd7c4408085f7c5584cdeebd1953c539972f80c5ef29bc147008630e
+R 34a870a8db0fd3ad276f368e06b50630
 U drh
-Z 445b2c6d99e984c3a2a0aca154b718db
+Z 3aa22be27e86e8143572e422fdd4e2d3
diff --git a/manifest.uuid b/manifest.uuid
index ddc6bd32ec..e16faf1fed 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-831156a4bd7c4408085f7c5584cdeebd1953c539972f80c5ef29bc147008630e
\ No newline at end of file
+28a5aec118f9d078c9e7225f85cd48a91920b13908c540771e309557c3f15f84
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 074041b3f0..cb3172e076 100644
--- a/src/build.c
+++ b/src/build.c
@@ -3883,8 +3883,10 @@ SrcList *sqlite3SrcListAppendFromTerm(
 */
 void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
   assert( pIndexedBy!=0 );
-  if( p && ALWAYS(p->nSrc>0) ){
-    struct SrcList_item *pItem = &p->a[p->nSrc-1];
+  if( p && pIndexedBy->n>0 ){
+    struct SrcList_item *pItem;
+    assert( p->nSrc>0 );
+    pItem = &p->a[p->nSrc-1];
     assert( pItem->fg.notIndexed==0 );
     assert( pItem->fg.isIndexedBy==0 );
     assert( pItem->fg.isTabFunc==0 );
@@ -3894,7 +3896,7 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
       pItem->fg.notIndexed = 1;
     }else{
       pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy);
-      pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0);
+      pItem->fg.isIndexedBy = 1;
     }
   }
 }

From 3f18e6d7a93d36ec7a3b29054f0bd7c1d85d64e9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 12 Aug 2017 02:01:55 +0000
Subject: [PATCH 002/270] Remove the zBase field from the StrAccum object. 
 Resulting code is slightly smaller and faster.

FossilOrigin-Name: 6e52fa5fd79988a433bae0152ceae036edab4bb18d2b48ed04c1f53f141728b0
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/printf.c    | 19 ++++++++-----------
 src/sqliteInt.h |  3 +--
 4 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/manifest b/manifest
index 10e55ba5d4..126340c6aa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\sand\sperformance\smicro-optimization\son\ssqlite3SrcListIndexedBy().
-D 2017-08-12T01:09:06.103
+C Remove\sthe\szBase\sfield\sfrom\sthe\sStrAccum\sobject.\s\sResulting\scode\sis\sslightly\nsmaller\sand\sfaster.
+D 2017-08-12T02:01:55.477
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -449,7 +449,7 @@ F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
 F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 3cbb99757d7295997674972f9dd2331c5c544368854ca08954c9beb1e9b6145a
-F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
+F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -459,7 +459,7 @@ F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c17
 F src/sqlite.h.in 72f1775c7a134f9e358eedafe1ebc703c28b0d705d976464ddbf6a9219448952
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 07e4d3c8021aea80e3bbafab4dd52833cfcfa4f000210af0d15c7fdaed2f09fc
+F src/sqliteInt.h 5e3c160c1e97568d72a5b2b755cd899d32a96f43a2a58eb18ad83da33b77a0cb
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1646,7 +1646,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 831156a4bd7c4408085f7c5584cdeebd1953c539972f80c5ef29bc147008630e
-R 34a870a8db0fd3ad276f368e06b50630
+P 28a5aec118f9d078c9e7225f85cd48a91920b13908c540771e309557c3f15f84
+R c12bc93088f8c532a7cb23532de2f4c4
 U drh
-Z 3aa22be27e86e8143572e422fdd4e2d3
+Z fd24d8fda45b9e8f917fec0b522deb6b
diff --git a/manifest.uuid b/manifest.uuid
index e16faf1fed..887a009764 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-28a5aec118f9d078c9e7225f85cd48a91920b13908c540771e309557c3f15f84
\ No newline at end of file
+6e52fa5fd79988a433bae0152ceae036edab4bb18d2b48ed04c1f53f141728b0
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index a14e658875..49b13cc4f5 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -782,7 +782,6 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){
   }else{
     char *zOld = isMalloced(p) ? p->zText : 0;
     i64 szNew = p->nChar;
-    assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) );
     szNew += N + 1;
     if( szNew+p->nChar<=p->mxAlloc ){
       /* Force exponential buffer size growth as long as it does not overflow,
@@ -824,7 +823,6 @@ void sqlite3AppendChar(StrAccum *p, int N, char c){
   if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){
     return;
   }
-  assert( (p->zText==p->zBase)==!isMalloced(p) );
   while( (N--)>0 ) p->zText[p->nChar++] = c;
 }
 
@@ -842,7 +840,6 @@ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){
     memcpy(&p->zText[p->nChar], z, N);
     p->nChar += N;
   }
-  assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) );
 }
 
 /*
@@ -877,19 +874,20 @@ void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){
 ** pointer if any kind of error was encountered.
 */
 static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){
+  char *zText;
   assert( p->mxAlloc>0 && !isMalloced(p) );
-  p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 );
-  if( p->zText ){
-    memcpy(p->zText, p->zBase, p->nChar+1);
+  zText = sqlite3DbMallocRaw(p->db, p->nChar+1 );
+  if( zText ){
+    memcpy(zText, p->zText, p->nChar+1);
     p->printfFlags |= SQLITE_PRINTF_MALLOCED;
   }else{
     setStrAccumError(p, STRACCUM_NOMEM);
   }
-  return p->zText;
+  p->zText = zText;
+  return zText;
 }
 char *sqlite3StrAccumFinish(StrAccum *p){
   if( p->zText ){
-    assert( (p->zText==p->zBase)==!isMalloced(p) );
     p->zText[p->nChar] = 0;
     if( p->mxAlloc>0 && !isMalloced(p) ){
       return strAccumFinishRealloc(p);
@@ -902,7 +900,6 @@ char *sqlite3StrAccumFinish(StrAccum *p){
 ** Reset an StrAccum string.  Reclaim all malloced memory.
 */
 void sqlite3StrAccumReset(StrAccum *p){
-  assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) );
   if( isMalloced(p) ){
     sqlite3DbFree(p->db, p->zText);
     p->printfFlags &= ~SQLITE_PRINTF_MALLOCED;
@@ -925,11 +922,11 @@ void sqlite3StrAccumReset(StrAccum *p){
 **        allocations will ever occur.
 */
 void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){
-  p->zText = p->zBase = zBase;
+  p->zText = zBase;
   p->db = db;
-  p->nChar = 0;
   p->nAlloc = n;
   p->mxAlloc = mx;
+  p->nChar = 0;
   p->accError = 0;
   p->printfFlags = 0;
 }
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 7222fcda94..522ab96517 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3227,11 +3227,10 @@ struct DbFixer {
 */
 struct StrAccum {
   sqlite3 *db;         /* Optional database for lookaside.  Can be NULL */
-  char *zBase;         /* A base allocation.  Not from malloc. */
   char *zText;         /* The string collected so far */
-  u32  nChar;          /* Length of the string so far */
   u32  nAlloc;         /* Amount of space allocated in zText */
   u32  mxAlloc;        /* Maximum allowed allocation.  0 for no malloc usage */
+  u32  nChar;          /* Length of the string so far */
   u8   accError;       /* STRACCUM_NOMEM or STRACCUM_TOOBIG */
   u8   printfFlags;    /* SQLITE_PRINTF flags below */
 };

From dccf4f2b15061cb84bee3c6c3c874edba563be58 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 12 Aug 2017 02:16:34 +0000
Subject: [PATCH 003/270] Update the speed-check.sh test script to append log
 output to the end of the cout-NAME.txt file.

FossilOrigin-Name: 14d262d6aa4e281dfe0490988f0c1965c4babf98038a1a96b9bb5772a61521a3
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/speed-check.sh |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 126340c6aa..9a39ea574b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\szBase\sfield\sfrom\sthe\sStrAccum\sobject.\s\sResulting\scode\sis\sslightly\nsmaller\sand\sfaster.
-D 2017-08-12T02:01:55.477
+C Update\sthe\sspeed-check.sh\stest\sscript\sto\sappend\slog\soutput\sto\sthe\send\sof\sthe\ncout-NAME.txt\sfile.
+D 2017-08-12T02:16:34.150
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -1606,7 +1606,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl f40dc82b4d5e39d040a02a3ec38268e324068815e4292a15ffa30ee93208bbfd
-F tool/speed-check.sh fd24151fd66465f01886c3f75faf8fa46d19f068c69d16514ca73887adcdafe4
+F tool/speed-check.sh 9eccb9ade8806238a4e9d6cb6511e7be2f64aff6873c41ea70d322219ea28adf
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1646,7 +1646,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 28a5aec118f9d078c9e7225f85cd48a91920b13908c540771e309557c3f15f84
-R c12bc93088f8c532a7cb23532de2f4c4
+P 6e52fa5fd79988a433bae0152ceae036edab4bb18d2b48ed04c1f53f141728b0
+R 772c60eb3b2c75ff30436857fe8c0a95
 U drh
-Z fd24d8fda45b9e8f917fec0b522deb6b
+Z 259464aafd26637749328aadb6f43b1b
diff --git a/manifest.uuid b/manifest.uuid
index 887a009764..5c0e0690f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6e52fa5fd79988a433bae0152ceae036edab4bb18d2b48ed04c1f53f141728b0
\ No newline at end of file
+14d262d6aa4e281dfe0490988f0c1965c4babf98038a1a96b9bb5772a61521a3
\ No newline at end of file
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
index 5de218748a..f9efb7c13d 100644
--- a/tool/speed-check.sh
+++ b/tool/speed-check.sh
@@ -147,6 +147,8 @@ size sqlite3.o | tee -a summary-$NAME.txt
 wc sqlite3.c
 if test $doCachegrind -eq 1; then
   cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
+  echo '*****************************************************' >>cout-$NAME.txt
+  sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt
 fi
 if test $doExplain -eq 1; then
   ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt

From 3c77a1e9f161aaf9fce41269ea4293fa890d0b45 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 12 Aug 2017 18:31:31 +0000
Subject: [PATCH 004/270] Add new test file fts5vocab2.test.

FossilOrigin-Name: 02174842c353bfaa747019cb3dcdee5bca6551d0a06d83fc1ac6d4569e16bc34
---
 ext/fts5/test/fts5vocab2.test | 209 ++++++++++++++++++++++++++++++++++
 manifest                      |  13 ++-
 manifest.uuid                 |   2 +-
 3 files changed, 217 insertions(+), 7 deletions(-)
 create mode 100644 ext/fts5/test/fts5vocab2.test

diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test
new file mode 100644
index 0000000000..4a0a1f4e3d
--- /dev/null
+++ b/ext/fts5/test/fts5vocab2.test
@@ -0,0 +1,209 @@
+# 2017 August 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The tests in this file focus on testing the fts5vocab module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5vocab
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM v1;
+} {
+  five  2 b 1
+  five  2 b 2
+  five  2 b 3
+  four  2 a 1
+  four  2 b 0
+  one   1 a 0
+  three 1 b 1
+  three 2 a 0
+  two   1 a 1
+  two   1 b 0
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 b 1
+  three 2 a 0
+}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 a 0
+  three 1 b 1
+  two   1 a 1
+  two   1 b 0
+}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 b 1
+  five  2 b 2
+  five  2 b 3
+  four  2 a 1
+  four  2 b 0
+  three 2 a 0
+}
+
+do_execsql_test 1.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS v1;
+
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 2.1 {
+  SELECT * FROM v1;
+} {
+  five  2 b {}
+  four  2 a {}
+  four  2 b {}
+  one   1 a {}
+  three 1 b {}
+  three 2 a {}
+  two   1 a {}
+  two   1 b {}
+}
+
+do_execsql_test 2.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 b {}
+  three 2 a {}
+}
+
+do_execsql_test 2.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 a {}
+  three 1 b {}
+  two   1 a {}
+  two   1 b {}
+}
+
+do_execsql_test 2.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 b {}
+  four  2 a {}
+  four  2 b {}
+  three 2 a {}
+}
+
+do_execsql_test 2.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS v1;
+
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
+
+  INSERT INTO t1 VALUES('one two', 'two three');
+  INSERT INTO t1 VALUES('three four', 'four five five five');
+}
+
+do_execsql_test 3.1 {
+  SELECT * FROM v1;
+} {
+  five  2 {} {}
+  four  2 {} {}
+  one   1 {} {}
+  three 1 {} {}
+  three 2 {} {}
+  two   1 {} {}
+}
+
+do_execsql_test 3.2 {
+  SELECT * FROM v1 WHERE term='three';
+} {
+  three 1 {} {}
+  three 2 {} {}
+}
+
+do_execsql_test 3.3 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=2;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  one   1 {} {}
+  three 1 {} {}
+  two   1 {} {}
+}
+
+do_execsql_test 3.4 {
+  BEGIN;
+    DELETE FROM t1 WHERE rowid=1;
+    SELECT * FROM v1;
+  ROLLBACK;
+} {
+  five  2 {} {}
+  four  2 {} {}
+  three 2 {} {}
+}
+
+do_execsql_test 3.5 {
+  DELETE FROM t1;
+  SELECT * FROM v1;
+} {
+}
+
+finish_test
+
diff --git a/manifest b/manifest
index 9a39ea574b..8fbd0a2dcd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sspeed-check.sh\stest\sscript\sto\sappend\slog\soutput\sto\sthe\send\sof\sthe\ncout-NAME.txt\sfile.
-D 2017-08-12T02:16:34.150
+C Add\snew\stest\sfile\sfts5vocab2.test.
+D 2017-08-12T18:31:31.810
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -201,6 +201,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test 0737876e20e97a6a6abf45de19fc99315727bcee6a83fadcada1cc080b9aa8f0
 F ext/fts5/test/fts5version.test 99b81372630fbf359107c96580fa761e41cdfb1dafc9966e148629ca72efee71
 F ext/fts5/test/fts5vocab.test 2ba98bcef0fcab3e5fead8eaabd6c0efb7e57bfe707a5cfcc18572ca9b277360
+F ext/fts5/test/fts5vocab2.test 2beeec974a305a1d79b91426622cc922c87065874437d22b400de7438979959e
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -1646,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6e52fa5fd79988a433bae0152ceae036edab4bb18d2b48ed04c1f53f141728b0
-R 772c60eb3b2c75ff30436857fe8c0a95
-U drh
-Z 259464aafd26637749328aadb6f43b1b
+P 14d262d6aa4e281dfe0490988f0c1965c4babf98038a1a96b9bb5772a61521a3
+R 458c6571aa20c315f2d6b3e7fbde0c5a
+U dan
+Z d58fa1886079f8c44974d3b28910f9ac
diff --git a/manifest.uuid b/manifest.uuid
index 5c0e0690f9..ab91f374e9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-14d262d6aa4e281dfe0490988f0c1965c4babf98038a1a96b9bb5772a61521a3
\ No newline at end of file
+02174842c353bfaa747019cb3dcdee5bca6551d0a06d83fc1ac6d4569e16bc34
\ No newline at end of file

From 1112cc797b2b588287e32aaee537343a9d3dc02e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 14 Aug 2017 01:33:07 +0000
Subject: [PATCH 005/270] Properly dequote column names in tables constructed
 by an aggregate SELECT.

FossilOrigin-Name: 7e0d3e9cb071873564b7916c022aba5cd9f3b8ebab9dba787ecd7113c5b7816a
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/select.c      |  4 +++-
 test/colname.test | 17 +++++++++++++++++
 4 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 8fbd0a2dcd..1caf0479d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\stest\sfile\sfts5vocab2.test.
-D 2017-08-12T18:31:31.810
+C Properly\sdequote\scolumn\snames\sin\stables\sconstructed\sby\san\saggregate\sSELECT.
+D 2017-08-14T01:33:07.818
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -454,7 +454,7 @@ F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 3fd19c98c5223d411b883502d1ac928ddb762a1ea8f031d910210316545fc67c
+F src/select.c ea8921065512650a9df3f5bf9a9b531c6ef4fb193c0d57e09d7a479ef9b13992
 F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
 F src/sqlite.h.in 72f1775c7a134f9e358eedafe1ebc703c28b0d705d976464ddbf6a9219448952
@@ -656,7 +656,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test b111edd2a84f558567320904bb94c779d7eec47254265b5f0a3d1f3e52cc28e0
+F test/colname.test c47639d26cbeba6977457e5ef2c2c55c5b6c889478dd7eb0ed858ba894e7fa93
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 14d262d6aa4e281dfe0490988f0c1965c4babf98038a1a96b9bb5772a61521a3
-R 458c6571aa20c315f2d6b3e7fbde0c5a
-U dan
-Z d58fa1886079f8c44974d3b28910f9ac
+P 02174842c353bfaa747019cb3dcdee5bca6551d0a06d83fc1ac6d4569e16bc34
+R 81f3b44c4693553d4feb0c8d49d70bc5
+U drh
+Z d0554a39e56793c8309c789eafde6181
diff --git a/manifest.uuid b/manifest.uuid
index ab91f374e9..11e97f4b39 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02174842c353bfaa747019cb3dcdee5bca6551d0a06d83fc1ac6d4569e16bc34
\ No newline at end of file
+7e0d3e9cb071873564b7916c022aba5cd9f3b8ebab9dba787ecd7113c5b7816a
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 255d729223..98a64b6f42 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1710,7 +1710,9 @@ int sqlite3ColumnsFromExprList(
         pColExpr = pColExpr->pRight;
         assert( pColExpr!=0 );
       }
-      if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){
+      if( (pColExpr->op==TK_COLUMN || pColExpr->op==TK_AGG_COLUMN)
+       && pColExpr->pTab!=0 
+      ){
         /* For columns use the column name name */
         int iCol = pColExpr->iColumn;
         Table *pTab = pColExpr->pTab;
diff --git a/test/colname.test b/test/colname.test
index 1497fc275c..2e4ae89008 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -378,5 +378,22 @@ do_test colname-9.210 {
   execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3}
 } {a 1 n 3}
 
+# Make sure the quotation marks get removed from the column names
+# when constructing a new table from an aggregate SELECT.
+# Email from Juergen Palm on 2017-07-11.
+#
+do_execsql_test colname-10.100 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1("with space" TEXT);
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2 AS SELECT "with space" FROM t1;
+  PRAGMA table_info(t2);
+} {0 {with space} TEXT 0 {} 0}
+do_execsql_test colname-10.110 {
+  DROP TABLE IF EXISTS t3;
+  CREATE TABLE t3 AS SELECT "with space" FROM t1 GROUP BY 1;
+  PRAGMA table_info(t3);
+} {0 {with space} TEXT 0 {} 0}
+
 
 finish_test

From 4dd89d5a249e93cec34d4396f27b1e9c1eb9a296 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 14 Aug 2017 14:53:24 +0000
Subject: [PATCH 006/270] Sometimes a TK_COLUMN Expr node can have Expr.pTab==0
 if it is a reference to an expression column in an index on and expression. 
 Fix for ticket [aa98619ad08ddcab].

FossilOrigin-Name: d0da791ba0edfb65186459345e43500d8364a086a5a1651d828fecc1a1dd1edb
---
 manifest             | 18 +++++++++---------
 manifest.uuid        |  2 +-
 src/expr.c           |  2 +-
 src/select.c         |  2 ++
 src/sqliteInt.h      |  3 ++-
 test/indexexpr1.test | 22 ++++++++++++++++++++++
 6 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 1caf0479d4..0e89152592 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Properly\sdequote\scolumn\snames\sin\stables\sconstructed\sby\san\saggregate\sSELECT.
-D 2017-08-14T01:33:07.818
+C Sometimes\sa\sTK_COLUMN\sExpr\snode\scan\shave\sExpr.pTab==0\sif\sit\sis\sa\sreference\nto\san\sexpression\scolumn\sin\san\sindex\son\sand\sexpression.\s\sFix\sfor\sticket\n[aa98619ad08ddcab].
+D 2017-08-14T14:53:24.546
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -407,7 +407,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c fdb2fc465cabbf372fecad1fc2b291758bec74150b4db0fb945332e09df28a0e
+F src/expr.c dc436431dc50a0256b9dcd3daaa06aac0df21834f91068525f2eb3c10b9a7a9a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c ed8888ae80b39f5a5d403954e4a05e0a38303523dff8143161439c142d31dec1
@@ -454,13 +454,13 @@ F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c ea8921065512650a9df3f5bf9a9b531c6ef4fb193c0d57e09d7a479ef9b13992
+F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
 F src/sqlite.h.in 72f1775c7a134f9e358eedafe1ebc703c28b0d705d976464ddbf6a9219448952
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 5e3c160c1e97568d72a5b2b755cd899d32a96f43a2a58eb18ad83da33b77a0cb
+F src/sqliteInt.h 854a122ff0ebde410a66d4f967e9923de9002f73965c6c9fa0db544bf7e657d1
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -940,7 +940,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
-F test/indexexpr1.test f348668daf7f533f1e5578dd4f31e4b9a3875da1ee2a60a8d2d50b938a7699c9
+F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
 F test/indexexpr2.test 3ddd7f23bc381b9f2b7a15f2d083b1a4078e7733dce8295602ecfa3c74a34cf9
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02174842c353bfaa747019cb3dcdee5bca6551d0a06d83fc1ac6d4569e16bc34
-R 81f3b44c4693553d4feb0c8d49d70bc5
+P 7e0d3e9cb071873564b7916c022aba5cd9f3b8ebab9dba787ecd7113c5b7816a
+R fd85cb2e90a98ca349ba60eafe5856e3
 U drh
-Z d0554a39e56793c8309c789eafde6181
+Z 01f8f49b6a69b50c9cc82fb68a921316
diff --git a/manifest.uuid b/manifest.uuid
index 11e97f4b39..3f3bef9058 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7e0d3e9cb071873564b7916c022aba5cd9f3b8ebab9dba787ecd7113c5b7816a
\ No newline at end of file
+d0da791ba0edfb65186459345e43500d8364a086a5a1651d828fecc1a1dd1edb
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 6a3ccd833e..d090aab3b7 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1973,8 +1973,8 @@ int sqlite3ExprCanBeNull(const Expr *p){
     case TK_BLOB:
       return 0;
     case TK_COLUMN:
-      assert( p->pTab!=0 );
       return ExprHasProperty(p, EP_CanBeNull) ||
+             p->pTab==0 ||  /* Reference to column of index on expression */
              (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0);
     default:
       return 1;
diff --git a/src/select.c b/src/select.c
index 98a64b6f42..aedbcc4b3a 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1617,6 +1617,8 @@ static void generateColumnNames(
     Expr *p = pEList->a[i].pExpr;
 
     assert( p!=0 );
+    assert( p->op!=TK_AGG_COLUMN );  /* Agg processing has not run yet */
+    assert( p->op!=TK_COLUMN || p->pTab!=0 ); /* Covering indexes not yet coded */
     if( pEList->a[i].zName ){
       /* An AS clause always takes first priority */
       char *zName = pEList->a[i].zName;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 522ab96517..c457b5401b 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2385,7 +2385,8 @@ struct Expr {
                          ** TK_COLUMN: the value of p5 for OP_Column
                          ** TK_AGG_FUNCTION: nesting depth */
   AggInfo *pAggInfo;     /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */
-  Table *pTab;           /* Table for TK_COLUMN expressions. */
+  Table *pTab;           /* Table for TK_COLUMN expressions.  Can be NULL
+                         ** for a column of an index on an expression */
 };
 
 /*
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index 85ec0df1f0..0e24c8066f 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -380,4 +380,26 @@ do_execsql_test indexexpr1-1300.1 {
   SELECT a FROM t1300 WHERE substr(b,4)='ess' COLLATE nocase ORDER BY +a;
 } {3 4}
 
+# Ticket https://sqlite.org/src/tktview/aa98619a
+# Assertion fault using an index on a constant
+#
+do_execsql_test indexexpr1-1400 {
+  CREATE TABLE t1400(x TEXT);
+  CREATE INDEX t1400x ON t1400(1);  -- Index on a constant
+  SELECT 1 IN (SELECT 2) FROM t1400;
+} {}
+do_execsql_test indexexpr1-1410 {
+  INSERT INTO t1400 VALUES('a'),('b');
+  SELECT 1 IN (SELECT 2) FROM t1400;
+} {0 0}
+do_execsql_test indexexpr1-1420 {
+  SELECT 1 IN (SELECT 2 UNION ALL SELECT 1) FROM t1400;
+} {1 1}
+do_execsql_test indexexpr1-1430 {
+  DROP INDEX t1400x;
+  CREATE INDEX t1400x ON t1400(abs(15+3));
+  SELECT abs(15+3) IN (SELECT 17 UNION ALL SELECT 18) FROM t1;
+} {1 1}
+
+
 finish_test

From f0357d8b2c5eada815416cc0de607e572cf2f8b6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 14 Aug 2017 17:03:58 +0000
Subject: [PATCH 007/270] Fix harmless indentation error.

FossilOrigin-Name: 25e92baaaeb9e8a2650b3083d3febf3661ecf1e05d9d24b26fe9f87a03bdd8fa
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 0e89152592..79eb7f408e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Sometimes\sa\sTK_COLUMN\sExpr\snode\scan\shave\sExpr.pTab==0\sif\sit\sis\sa\sreference\nto\san\sexpression\scolumn\sin\san\sindex\son\sand\sexpression.\s\sFix\sfor\sticket\n[aa98619ad08ddcab].
-D 2017-08-14T14:53:24.546
+C Fix\sharmless\sindentation\serror.
+D 2017-08-14T17:03:58.073
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -397,7 +397,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 1a17ba1a765d80c3ca39ce33ff55f92e1f51eb84bbbdab5377f11d36b1515fa1
+F src/btree.c 5a6efa29cc6b78f3151a64424bd2f3c28e0158019c45786635ef5ff79d94e850
 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
 F src/btreeInt.h 97700795edf8a43245720414798b7b29d8e465aef46bf301ffacd431910c0da1
 F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7e0d3e9cb071873564b7916c022aba5cd9f3b8ebab9dba787ecd7113c5b7816a
-R fd85cb2e90a98ca349ba60eafe5856e3
+P d0da791ba0edfb65186459345e43500d8364a086a5a1651d828fecc1a1dd1edb
+R 94ae43f22349d16aa74bde77a321d195
 U drh
-Z 01f8f49b6a69b50c9cc82fb68a921316
+Z ebdb69ac8faf71033c2cac4a21de62a6
diff --git a/manifest.uuid b/manifest.uuid
index 3f3bef9058..a86a3b6943 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d0da791ba0edfb65186459345e43500d8364a086a5a1651d828fecc1a1dd1edb
\ No newline at end of file
+25e92baaaeb9e8a2650b3083d3febf3661ecf1e05d9d24b26fe9f87a03bdd8fa
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 71e7769c1a..6602b3395b 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4960,7 +4960,7 @@ static int moveToRoot(BtCursor *pCur){
                         0, pCur->curPagerFlags);
     if( rc!=SQLITE_OK ){
       pCur->eState = CURSOR_INVALID;
-       return rc;
+      return rc;
     }
     pCur->iPage = 0;
     pCur->curIntKey = pCur->apPage[0]->intKey;

From 44548e7218faf4fdffa8ec4943895f169bdeab23 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 14 Aug 2017 18:13:52 +0000
Subject: [PATCH 008/270] Change the internal btree routine moveToRoot() to
 return SQLITE_EMPTY if the table is empty or if it has pgnoRoot==0.  This
 simplifies the callers, making them smaller and faster.  The SQLITE_EMPTY
 result code is intercepted and changed into SQLITE_OK before surfacing in an
 API.

FossilOrigin-Name: 240d57143d943eaddd5f7c2d473f47a1d29417e61d28142f70f3d960bb9b30df
---
 manifest        | 14 +++++-----
 manifest.uuid   |  2 +-
 src/btree.c     | 69 +++++++++++++++++++++++++------------------------
 src/sqlite.h.in |  2 +-
 4 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/manifest b/manifest
index 79eb7f408e..df3d92278a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\sindentation\serror.
-D 2017-08-14T17:03:58.073
+C Change\sthe\sinternal\sbtree\sroutine\smoveToRoot()\sto\sreturn\sSQLITE_EMPTY\sif\nthe\stable\sis\sempty\sor\sif\sit\shas\spgnoRoot==0.\s\sThis\ssimplifies\sthe\scallers,\nmaking\sthem\ssmaller\sand\sfaster.\s\sThe\sSQLITE_EMPTY\sresult\scode\sis\sintercepted\nand\schanged\sinto\sSQLITE_OK\sbefore\ssurfacing\sin\san\sAPI.
+D 2017-08-14T18:13:52.027
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -397,7 +397,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 5a6efa29cc6b78f3151a64424bd2f3c28e0158019c45786635ef5ff79d94e850
+F src/btree.c 43d1c5b335984abd3f9d38e87305bb0da63a638d29ea35744aabad2ddbf9fa4d
 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
 F src/btreeInt.h 97700795edf8a43245720414798b7b29d8e465aef46bf301ffacd431910c0da1
 F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
@@ -457,7 +457,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
-F src/sqlite.h.in 72f1775c7a134f9e358eedafe1ebc703c28b0d705d976464ddbf6a9219448952
+F src/sqlite.h.in f0bd1abf5e27bd22b3bcaae2a861c1efc4ab7e752bf7eb102355135617eb8199
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h 854a122ff0ebde410a66d4f967e9923de9002f73965c6c9fa0db544bf7e657d1
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d0da791ba0edfb65186459345e43500d8364a086a5a1651d828fecc1a1dd1edb
-R 94ae43f22349d16aa74bde77a321d195
+P 25e92baaaeb9e8a2650b3083d3febf3661ecf1e05d9d24b26fe9f87a03bdd8fa
+R 842146c033a5d3f4b14318bddfc60ebc
 U drh
-Z ebdb69ac8faf71033c2cac4a21de62a6
+Z 43daddc7888a0248763a7ce6f2befced
diff --git a/manifest.uuid b/manifest.uuid
index a86a3b6943..2c60456962 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25e92baaaeb9e8a2650b3083d3febf3661ecf1e05d9d24b26fe9f87a03bdd8fa
\ No newline at end of file
+240d57143d943eaddd5f7c2d473f47a1d29417e61d28142f70f3d960bb9b30df
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 6602b3395b..c1f8cd9364 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4914,9 +4914,9 @@ static void moveToParent(BtCursor *pCur){
 ** single child page. This can only happen with the table rooted at page 1.
 **
 ** If the b-tree structure is empty, the cursor state is set to 
-** CURSOR_INVALID. Otherwise, the cursor is set to point to the first
-** cell located on the root (or virtual root) page and the cursor state
-** is set to CURSOR_VALID.
+** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise,
+** the cursor is set to point to the first cell located on the root
+** (or virtual root) page and the cursor state is set to CURSOR_VALID.
 **
 ** If this function returns successfully, it may be assumed that the
 ** page-header flags indicate that the [virtual] root-page is the expected 
@@ -4935,6 +4935,7 @@ static int moveToRoot(BtCursor *pCur){
   assert( CURSOR_VALID   < CURSOR_REQUIRESEEK );
   assert( CURSOR_FAULT   > CURSOR_REQUIRESEEK );
   assert( pCur->eState < CURSOR_REQUIRESEEK || pCur->iPage<0 );
+  assert( pCur->pgnoRoot>0 || pCur->iPage<0 );
 
   if( pCur->iPage>=0 ){
     if( pCur->iPage ){
@@ -4946,7 +4947,7 @@ static int moveToRoot(BtCursor *pCur){
     }
   }else if( pCur->pgnoRoot==0 ){
     pCur->eState = CURSOR_INVALID;
-    return SQLITE_OK;
+    return SQLITE_EMPTY;
   }else{
     assert( pCur->iPage==(-1) );
     if( pCur->eState>=CURSOR_REQUIRESEEK ){
@@ -4999,6 +5000,7 @@ skip_init:
     rc = moveToChild(pCur, subpage);
   }else{
     pCur->eState = CURSOR_INVALID;
+    rc = SQLITE_EMPTY;
   }
   return rc;
 }
@@ -5065,14 +5067,13 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
   rc = moveToRoot(pCur);
   if( rc==SQLITE_OK ){
-    if( pCur->eState==CURSOR_INVALID ){
-      assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
-      *pRes = 1;
-    }else{
-      assert( pCur->apPage[pCur->iPage]->nCell>0 );
-      *pRes = 0;
-      rc = moveToLeftmost(pCur);
-    }
+    assert( pCur->apPage[pCur->iPage]->nCell>0 );
+    *pRes = 0;
+    rc = moveToLeftmost(pCur);
+  }else if( rc==SQLITE_EMPTY ){
+    assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+    *pRes = 1;
+    rc = SQLITE_OK;
   }
   return rc;
 }
@@ -5104,20 +5105,18 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){
 
   rc = moveToRoot(pCur);
   if( rc==SQLITE_OK ){
-    if( CURSOR_INVALID==pCur->eState ){
-      assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
-      *pRes = 1;
+    assert( pCur->eState==CURSOR_VALID );
+    *pRes = 0;
+    rc = moveToRightmost(pCur);
+    if( rc==SQLITE_OK ){
+      pCur->curFlags |= BTCF_AtLast;
     }else{
-      assert( pCur->eState==CURSOR_VALID );
-      *pRes = 0;
-      rc = moveToRightmost(pCur);
-      if( rc==SQLITE_OK ){
-        pCur->curFlags |= BTCF_AtLast;
-      }else{
-        pCur->curFlags &= ~BTCF_AtLast;
-      }
-   
+      pCur->curFlags &= ~BTCF_AtLast;
     }
+  }else if( rc==SQLITE_EMPTY ){
+    assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+    *pRes = 1;
+    rc = SQLITE_OK;
   }
   return rc;
 }
@@ -5216,16 +5215,17 @@ int sqlite3BtreeMovetoUnpacked(
 
   rc = moveToRoot(pCur);
   if( rc ){
+    if( rc==SQLITE_EMPTY ){
+      assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+      *pRes = -1;
+      return SQLITE_OK;
+    }
     return rc;
   }
-  assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] );
-  assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit );
-  assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 );
-  if( pCur->eState==CURSOR_INVALID ){
-    *pRes = -1;
-    assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
-    return SQLITE_OK;
-  }
+  assert( pCur->apPage[pCur->iPage] );
+  assert( pCur->apPage[pCur->iPage]->isInit );
+  assert( pCur->eState==CURSOR_VALID );
+  assert( pCur->apPage[pCur->iPage]->nCell > 0 );
   assert( pCur->apPage[0]->intKey==pCur->curIntKey );
   assert( pCur->curIntKey || pIdxKey );
   for(;;){
@@ -8439,6 +8439,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
         btreeReleaseAllCursorPages(pCur);
         pCur->eState = CURSOR_REQUIRESEEK;
       }
+      if( rc==SQLITE_EMPTY ) rc = SQLITE_OK;
     }
   }
   return rc;
@@ -8903,11 +8904,11 @@ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){
   i64 nEntry = 0;                      /* Value to return in *pnEntry */
   int rc;                              /* Return code */
 
-  if( pCur->pgnoRoot==0 ){
+  rc = moveToRoot(pCur);
+  if( rc==SQLITE_EMPTY ){
     *pnEntry = 0;
     return SQLITE_OK;
   }
-  rc = moveToRoot(pCur);
 
   /* Unless an error occurs, the following loop runs one iteration for each
   ** page in the B-Tree structure (not including overflow pages). 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index ad97b9a0de..1020e5f3d6 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -432,7 +432,7 @@ int sqlite3_exec(
 #define SQLITE_FULL        13   /* Insertion failed because database is full */
 #define SQLITE_CANTOPEN    14   /* Unable to open the database file */
 #define SQLITE_PROTOCOL    15   /* Database lock protocol error */
-#define SQLITE_EMPTY       16   /* Not used */
+#define SQLITE_EMPTY       16   /* Internal use only */
 #define SQLITE_SCHEMA      17   /* The database schema changed */
 #define SQLITE_TOOBIG      18   /* String or BLOB exceeds size limit */
 #define SQLITE_CONSTRAINT  19   /* Abort due to constraint violation */

From f38dd3b68fd90a1e8d913c242bb50d3d9c42298a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 14 Aug 2017 23:53:02 +0000
Subject: [PATCH 009/270] Minor size and performance optimization to
 sqlite3BtreeCloseCursor().

FossilOrigin-Name: 16969338841734d00ab906a94b82480c7e1e426eb0ddf7b9e6aed722aee5d91f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index df3d92278a..d114803f0f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sinternal\sbtree\sroutine\smoveToRoot()\sto\sreturn\sSQLITE_EMPTY\sif\nthe\stable\sis\sempty\sor\sif\sit\shas\spgnoRoot==0.\s\sThis\ssimplifies\sthe\scallers,\nmaking\sthem\ssmaller\sand\sfaster.\s\sThe\sSQLITE_EMPTY\sresult\scode\sis\sintercepted\nand\schanged\sinto\sSQLITE_OK\sbefore\ssurfacing\sin\san\sAPI.
-D 2017-08-14T18:13:52.027
+C Minor\ssize\sand\sperformance\soptimization\sto\ssqlite3BtreeCloseCursor().
+D 2017-08-14T23:53:02.259
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -397,7 +397,7 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 43d1c5b335984abd3f9d38e87305bb0da63a638d29ea35744aabad2ddbf9fa4d
+F src/btree.c 05781141fe24e9e24719a1d9c9b6ce38480af115f85a8a26f389a089888060d7
 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
 F src/btreeInt.h 97700795edf8a43245720414798b7b29d8e465aef46bf301ffacd431910c0da1
 F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 25e92baaaeb9e8a2650b3083d3febf3661ecf1e05d9d24b26fe9f87a03bdd8fa
-R 842146c033a5d3f4b14318bddfc60ebc
+P 240d57143d943eaddd5f7c2d473f47a1d29417e61d28142f70f3d960bb9b30df
+R a7c4eab7d8d2ce1f319f583d46cbd640
 U drh
-Z 43daddc7888a0248763a7ce6f2befced
+Z e48f5e9e2a338851fd6ede62f6ca96de
diff --git a/manifest.uuid b/manifest.uuid
index 2c60456962..d88c8647b6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-240d57143d943eaddd5f7c2d473f47a1d29417e61d28142f70f3d960bb9b30df
\ No newline at end of file
+16969338841734d00ab906a94b82480c7e1e426eb0ddf7b9e6aed722aee5d91f
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index c1f8cd9364..dcd70b9283 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4296,7 +4296,6 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
     int i;
     BtShared *pBt = pCur->pBt;
     sqlite3BtreeEnter(pBtree);
-    sqlite3BtreeClearCursor(pCur);
     assert( pBt->pCursor!=0 );
     if( pBt->pCursor==pCur ){
       pBt->pCursor = pCur->pNext;
@@ -4315,7 +4314,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
     }
     unlockBtreeIfUnused(pBt);
     sqlite3_free(pCur->aOverflow);
-    /* sqlite3_free(pCur); */
+    sqlite3_free(pCur->pKey);
     sqlite3BtreeLeave(pBtree);
   }
   return SQLITE_OK;

From 352a35abf5bdd40b50d7344f34e57ad616352d80 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 15 Aug 2017 03:46:47 +0000
Subject: [PATCH 010/270] Btree optimization:  New field BtCursor.pPage that
 points to the current page, saving a single pointer dereference on each
 access.

FossilOrigin-Name: 373b71d19cad785922d5a80828f2fee0cbe7dff6594743e625bbdfa31b1ca131
---
 manifest       |  14 ++---
 manifest.uuid  |   2 +-
 src/btree.c    | 168 +++++++++++++++++++++++++++----------------------
 src/btreeInt.h |   3 +-
 4 files changed, 103 insertions(+), 84 deletions(-)

diff --git a/manifest b/manifest
index d114803f0f..3ee6a346e3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\ssize\sand\sperformance\soptimization\sto\ssqlite3BtreeCloseCursor().
-D 2017-08-14T23:53:02.259
+C Btree\soptimization:\s\sNew\sfield\sBtCursor.pPage\sthat\spoints\sto\sthe\scurrent\spage,\nsaving\sa\ssingle\spointer\sdereference\son\seach\saccess.
+D 2017-08-15T03:46:47.011
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -397,9 +397,9 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 05781141fe24e9e24719a1d9c9b6ce38480af115f85a8a26f389a089888060d7
+F src/btree.c d2f5f347e56f8b7ed1bb798087045a3b98cd63f45fde7675fd24b9e88b61304d
 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
-F src/btreeInt.h 97700795edf8a43245720414798b7b29d8e465aef46bf301ffacd431910c0da1
+F src/btreeInt.h f78671f594dafd88cf9a81253da04db81272b382d2dc074bb983d348b95d9d2d
 F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
 F src/callback.c 930648a084a3adc741c6471adfbdc50ba47ba3542421cb80a26f259f467de65e
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 240d57143d943eaddd5f7c2d473f47a1d29417e61d28142f70f3d960bb9b30df
-R a7c4eab7d8d2ce1f319f583d46cbd640
+P 16969338841734d00ab906a94b82480c7e1e426eb0ddf7b9e6aed722aee5d91f
+R 3d6cdbe8a873abb5b839027115a07163
 U drh
-Z e48f5e9e2a338851fd6ede62f6ca96de
+Z 53459ac0c08285b40b84acbd692920bc
diff --git a/manifest.uuid b/manifest.uuid
index d88c8647b6..d331463c6a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-16969338841734d00ab906a94b82480c7e1e426eb0ddf7b9e6aed722aee5d91f
\ No newline at end of file
+373b71d19cad785922d5a80828f2fee0cbe7dff6594743e625bbdfa31b1ca131
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index dcd70b9283..50981b0575 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -439,7 +439,8 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);  /* Forward reference */
+static void releasePage(MemPage *pPage);
+static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
 /*
 ***** This routine is used inside of assert() only ****
@@ -598,11 +599,13 @@ static void btreeClearHasContent(BtShared *pBt){
 */
 static void btreeReleaseAllCursorPages(BtCursor *pCur){
   int i;
-  for(i=0; i<=pCur->iPage; i++){
-    releasePage(pCur->apPage[i]);
-    pCur->apPage[i] = 0;
+  if( pCur->iPage>=0 ){
+    for(i=0; i<pCur->iPage; i++){
+      releasePageNotNull(pCur->apPage[i]);
+    }
+    releasePageNotNull(pCur->pPage);
+    pCur->iPage = -1;
   }
-  pCur->iPage = -1;
 }
 
 /*
@@ -771,7 +774,7 @@ static int btreeMoveto(
     if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT;
     sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey);
     if( pIdxKey->nField==0 ){
-      rc = SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno);
+      rc = SQLITE_CORRUPT;
       goto moveto_done;
     }
   }else{
@@ -2049,7 +2052,7 @@ static int getAndInitPage(
   int rc;
   DbPage *pDbPage;
   assert( sqlite3_mutex_held(pBt->mutex) );
-  assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] );
+  assert( pCur==0 || ppPage==&pCur->pPage );
   assert( pCur==0 || bReadOnly==pCur->curPagerFlags );
   assert( pCur==0 || pCur->iPage>0 );
 
@@ -2083,7 +2086,10 @@ static int getAndInitPage(
   return SQLITE_OK;
 
 getAndInitPage_error:
-  if( pCur ) pCur->iPage--;
+  if( pCur ){
+    pCur->iPage--;
+    pCur->pPage = pCur->apPage[pCur->iPage];
+  }
   testcase( pgno==0 );
   assert( pgno!=0 || rc==SQLITE_CORRUPT );
   return rc;
@@ -4293,7 +4299,6 @@ void sqlite3BtreeCursorZero(BtCursor *p){
 int sqlite3BtreeCloseCursor(BtCursor *pCur){
   Btree *pBtree = pCur->pBtree;
   if( pBtree ){
-    int i;
     BtShared *pBt = pCur->pBt;
     sqlite3BtreeEnter(pBtree);
     assert( pBt->pCursor!=0 );
@@ -4309,9 +4314,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
         pPrev = pPrev->pNext;
       }while( ALWAYS(pPrev) );
     }
-    for(i=0; i<=pCur->iPage; i++){
-      releasePageNotNull(pCur->apPage[i]);
-    }
+    btreeReleaseAllCursorPages(pCur);
     unlockBtreeIfUnused(pBt);
     sqlite3_free(pCur->aOverflow);
     sqlite3_free(pCur->pKey);
@@ -4331,9 +4334,8 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
 #ifndef NDEBUG
   static void assertCellInfo(BtCursor *pCur){
     CellInfo info;
-    int iPage = pCur->iPage;
     memset(&info, 0, sizeof(info));
-    btreeParseCell(pCur->apPage[iPage], pCur->ix, &info);
+    btreeParseCell(pCur->pPage, pCur->ix, &info);
     assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 );
   }
 #else
@@ -4341,9 +4343,8 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
 #endif
 static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){
   if( pCur->info.nSize==0 ){
-    int iPage = pCur->iPage;
     pCur->curFlags |= BTCF_ValidNKey;
-    btreeParseCell(pCur->apPage[iPage],pCur->ix,&pCur->info);
+    btreeParseCell(pCur->pPage,pCur->ix,&pCur->info);
   }else{
     assertCellInfo(pCur);
   }
@@ -4541,7 +4542,7 @@ static int accessPayload(
   unsigned char *aPayload;
   int rc = SQLITE_OK;
   int iIdx = 0;
-  MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */
+  MemPage *pPage = pCur->pPage;               /* Btree page of current entry */
   BtShared *pBt = pCur->pBt;                  /* Btree this cursor belongs to */
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
   unsigned char * const pBufStart = pBuf;     /* Start of original out buffer */
@@ -4737,8 +4738,8 @@ static int accessPayload(
 int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){
   assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] );
-  assert( pCur->ix<pCur->apPage[pCur->iPage]->nCell );
+  assert( pCur->iPage>=0 && pCur->pPage );
+  assert( pCur->ix<pCur->pPage->nCell );
   return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0);
 }
 
@@ -4796,15 +4797,15 @@ static const void *fetchPayload(
   u32 *pAmt            /* Write the number of available bytes here */
 ){
   u32 amt;
-  assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]);
+  assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage);
   assert( pCur->eState==CURSOR_VALID );
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
   assert( cursorOwnsBtShared(pCur) );
-  assert( pCur->ix<pCur->apPage[pCur->iPage]->nCell );
+  assert( pCur->ix<pCur->pPage->nCell );
   assert( pCur->info.nSize>0 );
-  assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB );
-  assert( pCur->info.pPayload<pCur->apPage[pCur->iPage]->aDataEnd ||CORRUPT_DB);
-  amt = (int)(pCur->apPage[pCur->iPage]->aDataEnd - pCur->info.pPayload);
+  assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB );
+  assert( pCur->info.pPayload<pCur->pPage->aDataEnd ||CORRUPT_DB);
+  amt = (int)(pCur->pPage->aDataEnd - pCur->info.pPayload);
   if( pCur->info.nLocal<amt ) amt = pCur->info.nLocal;
   *pAmt = amt;
   return (void*)pCur->info.pPayload;
@@ -4851,10 +4852,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   }
   pCur->info.nSize = 0;
   pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
-  pCur->aiIdx[pCur->iPage++] = pCur->ix;
+  pCur->aiIdx[pCur->iPage] = pCur->ix;
+  pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
-  return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage],
-                        pCur, pCur->curPagerFlags);
+  pCur->iPage++;
+  return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags);
 }
 
 #ifdef SQLITE_DEBUG
@@ -4888,20 +4890,23 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){
 ** the largest cell index.
 */
 static void moveToParent(BtCursor *pCur){
+  MemPage *pLeaf;
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
   assert( pCur->iPage>0 );
-  assert( pCur->apPage[pCur->iPage] );
+  assert( pCur->pPage );
   assertParentIndex(
     pCur->apPage[pCur->iPage-1], 
     pCur->aiIdx[pCur->iPage-1], 
-    pCur->apPage[pCur->iPage]->pgno
+    pCur->pPage->pgno
   );
   testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell );
   pCur->info.nSize = 0;
   pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
   pCur->ix = pCur->aiIdx[pCur->iPage-1];
-  releasePageNotNull(pCur->apPage[pCur->iPage--]);
+  pLeaf = pCur->pPage;
+  pCur->pPage = pCur->apPage[--pCur->iPage];
+  releasePageNotNull(pLeaf);
 }
 
 /*
@@ -4938,10 +4943,11 @@ static int moveToRoot(BtCursor *pCur){
 
   if( pCur->iPage>=0 ){
     if( pCur->iPage ){
-      do{
-        assert( pCur->apPage[pCur->iPage]!=0 );
-        releasePageNotNull(pCur->apPage[pCur->iPage--]);
-      }while( pCur->iPage);
+      releasePageNotNull(pCur->pPage);
+      while( --pCur->iPage ){
+        releasePageNotNull(pCur->apPage[pCur->iPage]);
+      }
+      pCur->pPage = pCur->apPage[0];
       goto skip_init;
     }
   }else if( pCur->pgnoRoot==0 ){
@@ -4956,16 +4962,16 @@ static int moveToRoot(BtCursor *pCur){
       }
       sqlite3BtreeClearCursor(pCur);
     }
-    rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0],
+    rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage,
                         0, pCur->curPagerFlags);
     if( rc!=SQLITE_OK ){
       pCur->eState = CURSOR_INVALID;
       return rc;
     }
     pCur->iPage = 0;
-    pCur->curIntKey = pCur->apPage[0]->intKey;
+    pCur->curIntKey = pCur->pPage->intKey;
   }
-  pRoot = pCur->apPage[0];
+  pRoot = pCur->pPage;
   assert( pRoot->pgno==pCur->pgnoRoot );
 
   /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor
@@ -4980,7 +4986,7 @@ static int moveToRoot(BtCursor *pCur){
   ** (or the freelist).  */
   assert( pRoot->intKey==1 || pRoot->intKey==0 );
   if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){
-    return SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno);
+    return SQLITE_CORRUPT_PGNO(pCur->pPage->pgno);
   }
 
 skip_init:  
@@ -4988,7 +4994,7 @@ skip_init:
   pCur->info.nSize = 0;
   pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl);
 
-  pRoot = pCur->apPage[0];
+  pRoot = pCur->pPage;
   if( pRoot->nCell>0 ){
     pCur->eState = CURSOR_VALID;
   }else if( !pRoot->leaf ){
@@ -5018,7 +5024,7 @@ static int moveToLeftmost(BtCursor *pCur){
 
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){
+  while( rc==SQLITE_OK && !(pPage = pCur->pPage)->leaf ){
     assert( pCur->ix<pPage->nCell );
     pgno = get4byte(findCell(pPage, pCur->ix));
     rc = moveToChild(pCur, pgno);
@@ -5043,7 +5049,7 @@ static int moveToRightmost(BtCursor *pCur){
 
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){
+  while( !(pPage = pCur->pPage)->leaf ){
     pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]);
     pCur->ix = pPage->nCell;
     rc = moveToChild(pCur, pgno);
@@ -5066,11 +5072,11 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
   rc = moveToRoot(pCur);
   if( rc==SQLITE_OK ){
-    assert( pCur->apPage[pCur->iPage]->nCell>0 );
+    assert( pCur->pPage->nCell>0 );
     *pRes = 0;
     rc = moveToLeftmost(pCur);
   }else if( rc==SQLITE_EMPTY ){
-    assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+    assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 );
     *pRes = 1;
     rc = SQLITE_OK;
   }
@@ -5096,8 +5102,8 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){
     for(ii=0; ii<pCur->iPage; ii++){
       assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell );
     }
-    assert( pCur->ix==pCur->apPage[pCur->iPage]->nCell-1 );
-    assert( pCur->apPage[pCur->iPage]->leaf );
+    assert( pCur->ix==pCur->pPage->nCell-1 );
+    assert( pCur->pPage->leaf );
 #endif
     return SQLITE_OK;
   }
@@ -5113,7 +5119,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){
       pCur->curFlags &= ~BTCF_AtLast;
     }
   }else if( rc==SQLITE_EMPTY ){
-    assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+    assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 );
     *pRes = 1;
     rc = SQLITE_OK;
   }
@@ -5215,22 +5221,22 @@ int sqlite3BtreeMovetoUnpacked(
   rc = moveToRoot(pCur);
   if( rc ){
     if( rc==SQLITE_EMPTY ){
-      assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 );
+      assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 );
       *pRes = -1;
       return SQLITE_OK;
     }
     return rc;
   }
-  assert( pCur->apPage[pCur->iPage] );
-  assert( pCur->apPage[pCur->iPage]->isInit );
+  assert( pCur->pPage );
+  assert( pCur->pPage->isInit );
   assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->apPage[pCur->iPage]->nCell > 0 );
-  assert( pCur->apPage[0]->intKey==pCur->curIntKey );
+  assert( pCur->pPage->nCell > 0 );
+  assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey );
   assert( pCur->curIntKey || pIdxKey );
   for(;;){
     int lwr, upr, idx, c;
     Pgno chldPg;
-    MemPage *pPage = pCur->apPage[pCur->iPage];
+    MemPage *pPage = pCur->pPage;
     u8 *pCell;                          /* Pointer to current cell in pPage */
 
     /* pPage->nCell must be greater than zero. If this is the root-page
@@ -5369,7 +5375,7 @@ int sqlite3BtreeMovetoUnpacked(
     assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) );
     assert( pPage->isInit );
     if( pPage->leaf ){
-      assert( pCur->ix<pCur->apPage[pCur->iPage]->nCell );
+      assert( pCur->ix<pCur->pPage->nCell );
       pCur->ix = (u16)idx;
       *pRes = c;
       rc = SQLITE_OK;
@@ -5423,9 +5429,10 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){
   ** opcode, and it that case the cursor will always be valid and
   ** will always point to a leaf node. */
   if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1;
-  if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1;
+  if( NEVER(pCur->pPage->leaf==0) ) return -1;
 
-  for(n=1, i=0; i<=pCur->iPage; i++){
+  n = pCur->pPage->nCell;
+  for(i=0; i<pCur->iPage; i++){
     n *= pCur->apPage[i]->nCell;
   }
   return n;
@@ -5478,7 +5485,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){
     }
   }
 
-  pPage = pCur->apPage[pCur->iPage];
+  pPage = pCur->pPage;
   idx = ++pCur->ix;
   assert( pPage->isInit );
 
@@ -5501,7 +5508,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){
         return SQLITE_DONE;
       }
       moveToParent(pCur);
-      pPage = pCur->apPage[pCur->iPage];
+      pPage = pCur->pPage;
     }while( pCur->ix>=pPage->nCell );
     if( pPage->intKey ){
       return sqlite3BtreeNext(pCur, 0);
@@ -5524,7 +5531,7 @@ int sqlite3BtreeNext(BtCursor *pCur, int flags){
   pCur->info.nSize = 0;
   pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl);
   if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur);
-  pPage = pCur->apPage[pCur->iPage];
+  pPage = pCur->pPage;
   if( (++pCur->ix)>=pPage->nCell ){
     pCur->ix--;
     return btreeNext(pCur);
@@ -5583,7 +5590,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){
     }
   }
 
-  pPage = pCur->apPage[pCur->iPage];
+  pPage = pCur->pPage;
   assert( pPage->isInit );
   if( !pPage->leaf ){
     int idx = pCur->ix;
@@ -5602,7 +5609,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){
     assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 );
 
     pCur->ix--;
-    pPage = pCur->apPage[pCur->iPage];
+    pPage = pCur->pPage;
     if( pPage->intKey && !pPage->leaf ){
       rc = sqlite3BtreePrevious(pCur, 0);
     }else{
@@ -5620,7 +5627,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int flags){
   pCur->info.nSize = 0;
   if( pCur->eState!=CURSOR_VALID
    || pCur->ix==0
-   || pCur->apPage[pCur->iPage]->leaf==0
+   || pCur->pPage->leaf==0
   ){
     return btreePrevious(pCur);
   }
@@ -7934,7 +7941,7 @@ static int balance(BtCursor *pCur){
 
   do {
     int iPage = pCur->iPage;
-    MemPage *pPage = pCur->apPage[iPage];
+    MemPage *pPage = pCur->pPage;
 
     if( iPage==0 ){
       if( pPage->nOverflow ){
@@ -7950,7 +7957,9 @@ static int balance(BtCursor *pCur){
           pCur->iPage = 1;
           pCur->ix = 0;
           pCur->aiIdx[0] = 0;
-          assert( pCur->apPage[1]->nOverflow );
+          pCur->apPage[0] = pPage;
+          pCur->pPage = pCur->apPage[1];
+          assert( pCur->pPage->nOverflow );
         }
       }else{
         break;
@@ -8030,6 +8039,7 @@ static int balance(BtCursor *pCur){
       releasePage(pPage);
       pCur->iPage--;
       assert( pCur->iPage>=0 );
+      pCur->pPage = pCur->apPage[pCur->iPage];
     }
   }while( rc==SQLITE_OK );
 
@@ -8161,7 +8171,7 @@ int sqlite3BtreeInsert(
   }
   assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) );
 
-  pPage = pCur->apPage[pCur->iPage];
+  pPage = pCur->pPage;
   assert( pPage->intKey || pX->nKey>=0 );
   assert( pPage->leaf || !pPage->intKey );
 
@@ -8248,7 +8258,7 @@ int sqlite3BtreeInsert(
     ** fails. Internal data structure corruption will result otherwise. 
     ** Also, set the cursor state to invalid. This stops saveCursorPosition()
     ** from trying to save the current position of the cursor.  */
-    pCur->apPage[pCur->iPage]->nOverflow = 0;
+    pCur->pPage->nOverflow = 0;
     pCur->eState = CURSOR_INVALID;
     if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){
       btreeReleaseAllCursorPages(pCur);
@@ -8265,7 +8275,7 @@ int sqlite3BtreeInsert(
       pCur->nKey = pX->nKey;
     }
   }
-  assert( pCur->iPage<0 || pCur->apPage[pCur->iPage]->nOverflow==0 );
+  assert( pCur->iPage<0 || pCur->pPage->nOverflow==0 );
 
 end_insert:
   return rc;
@@ -8306,13 +8316,13 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   assert( pCur->curFlags & BTCF_WriteFlag );
   assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
   assert( !hasReadConflicts(p, pCur->pgnoRoot) );
-  assert( pCur->ix<pCur->apPage[pCur->iPage]->nCell );
+  assert( pCur->ix<pCur->pPage->nCell );
   assert( pCur->eState==CURSOR_VALID );
   assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 );
 
   iCellDepth = pCur->iPage;
   iCellIdx = pCur->ix;
-  pPage = pCur->apPage[iCellDepth];
+  pPage = pCur->pPage;
   pCell = findCell(pPage, iCellIdx);
 
   /* If the bPreserve flag is set to true, then the cursor position must
@@ -8378,11 +8388,16 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   ** node. The cell from the leaf node needs to be moved to the internal
   ** node to replace the deleted cell.  */
   if( !pPage->leaf ){
-    MemPage *pLeaf = pCur->apPage[pCur->iPage];
+    MemPage *pLeaf = pCur->pPage;
     int nCell;
-    Pgno n = pCur->apPage[iCellDepth+1]->pgno;
+    Pgno n;
     unsigned char *pTmp;
 
+    if( iCellDepth<pCur->iPage-1 ){
+      n = pCur->apPage[iCellDepth+1]->pgno;
+    }else{
+      n = pCur->pPage->pgno;
+    }
     pCell = findCell(pLeaf, pLeaf->nCell-1);
     if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT;
     nCell = pLeaf->xCellSize(pLeaf, pCell);
@@ -8414,16 +8429,19 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   ** well.  */
   rc = balance(pCur);
   if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){
+    releasePageNotNull(pCur->pPage);
+    pCur->iPage--;
     while( pCur->iPage>iCellDepth ){
       releasePage(pCur->apPage[pCur->iPage--]);
     }
+    pCur->pPage = pCur->apPage[pCur->iPage];
     rc = balance(pCur);
   }
 
   if( rc==SQLITE_OK ){
     if( bSkipnext ){
       assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) );
-      assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB );
+      assert( pPage==pCur->pPage || CORRUPT_DB );
       assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell );
       pCur->eState = CURSOR_SKIPNEXT;
       if( iCellIdx>=pPage->nCell ){
@@ -8920,7 +8938,7 @@ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){
     ** this page contains countable entries. Increment the entry counter
     ** accordingly.
     */
-    pPage = pCur->apPage[pCur->iPage];
+    pPage = pCur->pPage;
     if( pPage->leaf || !pPage->intKey ){
       nEntry += pPage->nCell;
     }
@@ -8943,10 +8961,10 @@ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){
           return moveToRoot(pCur);
         }
         moveToParent(pCur);
-      }while ( pCur->ix>=pCur->apPage[pCur->iPage]->nCell );
+      }while ( pCur->ix>=pCur->pPage->nCell );
 
       pCur->ix++;
-      pPage = pCur->apPage[pCur->iPage];
+      pPage = pCur->pPage;
     }
 
     /* Descend to the child node of the cell that the cursor currently 
@@ -9787,7 +9805,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){
               && pCsr->pBt->inTransaction==TRANS_WRITE );
   assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) );
   assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) );
-  assert( pCsr->apPage[pCsr->iPage]->intKey );
+  assert( pCsr->pPage->intKey );
 
   return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1);
 }
diff --git a/src/btreeInt.h b/src/btreeInt.h
index 77069783c7..f4e5f45fec 100644
--- a/src/btreeInt.h
+++ b/src/btreeInt.h
@@ -522,7 +522,8 @@ struct BtCursor {
   u16 ix;                   /* Current index for apPage[iPage] */
   u16 aiIdx[BTCURSOR_MAX_DEPTH-1];     /* Current index in apPage[i] */
   struct KeyInfo *pKeyInfo;            /* Arg passed to comparison function */
-  MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */
+  MemPage *pPage;                        /* Current page */
+  MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */
 };
 
 /*

From 6cd8c8c57a249feac637ea97f21a8d3de7670de3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 15 Aug 2017 14:14:36 +0000
Subject: [PATCH 011/270] Small size and performance in the OP_Column opcode.

FossilOrigin-Name: 2cf3f3de8a48465bd6b0af7763bfe905f3bb0151488f63c9ecc3147bcb345094
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 16 ++++++----------
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 3ee6a346e3..43be9dbc63 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Btree\soptimization:\s\sNew\sfield\sBtCursor.pPage\sthat\spoints\sto\sthe\scurrent\spage,\nsaving\sa\ssingle\spointer\sdereference\son\seach\saccess.
-D 2017-08-15T03:46:47.011
+C Small\ssize\sand\sperformance\sin\sthe\sOP_Column\sopcode.
+D 2017-08-15T14:14:36.511
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -522,7 +522,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 821b3edde2d17ec60da0617db1018a88f38634c359d22f3c8f7be10336c82636
+F src/vdbe.c ebfc41ca25465888b2dc2969a0059d0d563c2a36e31696352573198cdec51fad
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h ff2b7db0968d20e6184aee256d2e535d565f5a172e3588a78adb166a41fc4911
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 16969338841734d00ab906a94b82480c7e1e426eb0ddf7b9e6aed722aee5d91f
-R 3d6cdbe8a873abb5b839027115a07163
+P 373b71d19cad785922d5a80828f2fee0cbe7dff6594743e625bbdfa31b1ca131
+R 36da37536a24b366658dfab60814a38c
 U drh
-Z 53459ac0c08285b40b84acbd692920bc
+Z 16cc8098bdc3d63f9bef627e24ebd270
diff --git a/manifest.uuid b/manifest.uuid
index d331463c6a..f58cbd3506 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-373b71d19cad785922d5a80828f2fee0cbe7dff6594743e625bbdfa31b1ca131
\ No newline at end of file
+2cf3f3de8a48465bd6b0af7763bfe905f3bb0151488f63c9ecc3147bcb345094
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index d04a4f2c91..e44f06f3c9 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2388,7 +2388,6 @@ case OP_Column: {
   const u8 *zEndHdr; /* Pointer to first byte after the header */
   u32 offset;        /* Offset into the data */
   u64 offset64;      /* 64-bit offset */
-  u32 avail;         /* Number of bytes of available data */
   u32 t;             /* A type code from the record header */
   Mem *pReg;         /* PseudoTable input register */
 
@@ -2419,7 +2418,7 @@ case OP_Column: {
         pReg = &aMem[pC->uc.pseudoTableReg];
         assert( pReg->flags & MEM_Blob );
         assert( memIsValid(pReg) );
-        pC->payloadSize = pC->szRow = avail = pReg->n;
+        pC->payloadSize = pC->szRow = pReg->n;
         pC->aRow = (u8*)pReg->z;
       }else{
         sqlite3VdbeMemSetNull(pDest);
@@ -2431,14 +2430,11 @@ case OP_Column: {
       assert( pCrsr );
       assert( sqlite3BtreeCursorIsValid(pCrsr) );
       pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
-      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
-      assert( avail<=65536 );  /* Maximum page size is 64KiB */
-      if( pC->payloadSize <= (u32)avail ){
-        pC->szRow = pC->payloadSize;
-      }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &pC->szRow);
+      assert( pC->szRow<=pC->payloadSize );
+      assert( pC->szRow<=65536 );  /* Maximum page size is 64KiB */
+      if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
         goto too_big;
-      }else{
-        pC->szRow = avail;
       }
     }
     pC->cacheStatus = p->cacheCtr;
@@ -2447,7 +2443,7 @@ case OP_Column: {
     aOffset[0] = offset;
 
 
-    if( avail<offset ){      /*OPTIMIZATION-IF-FALSE*/
+    if( pC->szRow<offset ){      /*OPTIMIZATION-IF-FALSE*/
       /* pC->aRow does not have to hold the entire row, but it does at least
       ** need to cover the header of the record.  If pC->aRow does not contain
       ** the complete header, then set it to zero, forcing the header to be

From 95b225a46d53ab790ec88eff9ade14403c8fe3eb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 16 Aug 2017 11:04:22 +0000
Subject: [PATCH 012/270] Performance improvement in the OP_Column opcode.

FossilOrigin-Name: dc98a92f32511ee322b0207bd286e967248a8e59b418f11168eb31e34b0fa0fa
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 30 +++++++++++++++++++++++-------
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 43be9dbc63..251d1e672d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\ssize\sand\sperformance\sin\sthe\sOP_Column\sopcode.
-D 2017-08-15T14:14:36.511
+C Performance\simprovement\sin\sthe\sOP_Column\sopcode.
+D 2017-08-16T11:04:22.469
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -522,7 +522,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c ebfc41ca25465888b2dc2969a0059d0d563c2a36e31696352573198cdec51fad
+F src/vdbe.c 8530c38ffc19400cf9634d2e55b9c6141d276350e6ea6d99a003cd6963fbf20a
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h ff2b7db0968d20e6184aee256d2e535d565f5a172e3588a78adb166a41fc4911
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 373b71d19cad785922d5a80828f2fee0cbe7dff6594743e625bbdfa31b1ca131
-R 36da37536a24b366658dfab60814a38c
+P 2cf3f3de8a48465bd6b0af7763bfe905f3bb0151488f63c9ecc3147bcb345094
+R 1db09713c9077dbfc76327c90333340a
 U drh
-Z 16cc8098bdc3d63f9bef627e24ebd270
+Z 65fa1f244e3ef6aef2172aa886d1ccda
diff --git a/manifest.uuid b/manifest.uuid
index f58cbd3506..5e9b037d44 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2cf3f3de8a48465bd6b0af7763bfe905f3bb0151488f63c9ecc3147bcb345094
\ No newline at end of file
+dc98a92f32511ee322b0207bd286e967248a8e59b418f11168eb31e34b0fa0fa
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index e44f06f3c9..c7ef2ef6a1 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2464,13 +2464,23 @@ case OP_Column: {
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
-    }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/
-      /* The following goto is an optimization.  It can be omitted and
-      ** everything will still work.  But OP_Column is measurably faster
-      ** by skipping the subsequent conditional, which is always true.
+    }else{
+      /* This is an optimization.  By skipping over the first few tests
+      ** (ex: pC->nHdrParsed<=p2) in the next section, we achieve a
+      ** measurable performance gain.
+      **
+      ** This branch is taken even if offset==0.  Such a record is never
+      ** generated by SQLite, and could be considered corruption, but we
+      ** accept it for historical reasons.  When offset==0, the code this
+      ** branch jumps to reads past the end of the record, but never more
+      ** than a few bytes.  Even if the record occurs at the end of the page
+      ** content area, the "page header" comes after the page content and so
+      ** this overread is harmless.  Similar overreads can occur for a corrupt
+      ** database file.
       */
       zData = pC->aRow;
       assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
+      testcase( offset==0 );
       goto op_column_read_header;
     }
   }
@@ -2499,6 +2509,7 @@ case OP_Column: {
       offset64 = aOffset[i];
       zHdr = zData + pC->iHdrOffset;
       zEndHdr = zData + aOffset[0];
+      testcase( zHdr>=zEndHdr );
       do{
         if( (t = zHdr[0])<0x80 ){
           zHdr++;
@@ -2519,9 +2530,14 @@ case OP_Column: {
       if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize))
        || (offset64 > pC->payloadSize)
       ){
-        if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
-        rc = SQLITE_CORRUPT_BKPT;
-        goto abort_due_to_error;
+        if( aOffset[0]==0 ){
+          i = 0;
+          zHdr = zEndHdr;
+        }else{
+          if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
+          rc = SQLITE_CORRUPT_BKPT;
+          goto abort_due_to_error;
+        }
       }
 
       pC->nHdrParsed = i;

From 1f613c4df389356a3a679cca0066e9381e39467e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 16 Aug 2017 14:16:19 +0000
Subject: [PATCH 013/270] Remove an unnecessary local variable from OP_Column,
 for a small size reduction and performance increase.

FossilOrigin-Name: 39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 14 ++++++--------
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 251d1e672d..514232a7c7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Performance\simprovement\sin\sthe\sOP_Column\sopcode.
-D 2017-08-16T11:04:22.469
+C Remove\san\sunnecessary\slocal\svariable\sfrom\sOP_Column,\sfor\sa\ssmall\ssize\nreduction\sand\sperformance\sincrease.
+D 2017-08-16T14:16:19.879
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -522,7 +522,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 8530c38ffc19400cf9634d2e55b9c6141d276350e6ea6d99a003cd6963fbf20a
+F src/vdbe.c c8e7bec8764b26b8e80fadffab2b9c18e8ed5ab2e92d1be5a7aebeed18ca8be5
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h ff2b7db0968d20e6184aee256d2e535d565f5a172e3588a78adb166a41fc4911
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2cf3f3de8a48465bd6b0af7763bfe905f3bb0151488f63c9ecc3147bcb345094
-R 1db09713c9077dbfc76327c90333340a
+P dc98a92f32511ee322b0207bd286e967248a8e59b418f11168eb31e34b0fa0fa
+R 8dc9658e95fe266a68d077704d4afe1a
 U drh
-Z 65fa1f244e3ef6aef2172aa886d1ccda
+Z 4fdaa02107e5a271b3b6d501790c7a7c
diff --git a/manifest.uuid b/manifest.uuid
index 5e9b037d44..e7180c4267 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dc98a92f32511ee322b0207bd286e967248a8e59b418f11168eb31e34b0fa0fa
\ No newline at end of file
+39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index c7ef2ef6a1..a4665ce79c 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2386,7 +2386,6 @@ case OP_Column: {
   const u8 *zData;   /* Part of the record being decoded */
   const u8 *zHdr;    /* Next unparsed byte of the header */
   const u8 *zEndHdr; /* Pointer to first byte after the header */
-  u32 offset;        /* Offset into the data */
   u64 offset64;      /* 64-bit offset */
   u32 t;             /* A type code from the record header */
   Mem *pReg;         /* PseudoTable input register */
@@ -2438,12 +2437,11 @@ case OP_Column: {
       }
     }
     pC->cacheStatus = p->cacheCtr;
-    pC->iHdrOffset = getVarint32(pC->aRow, offset);
+    pC->iHdrOffset = getVarint32(pC->aRow, aOffset[0]);
     pC->nHdrParsed = 0;
-    aOffset[0] = offset;
 
 
-    if( pC->szRow<offset ){      /*OPTIMIZATION-IF-FALSE*/
+    if( pC->szRow<aOffset[0] ){      /*OPTIMIZATION-IF-FALSE*/
       /* pC->aRow does not have to hold the entire row, but it does at least
       ** need to cover the header of the record.  If pC->aRow does not contain
       ** the complete header, then set it to zero, forcing the header to be
@@ -2460,7 +2458,7 @@ case OP_Column: {
       ** 3-byte type for each of the maximum of 32768 columns plus three
       ** extra bytes for the header length itself.  32768*3 + 3 = 98307.
       */
-      if( offset > 98307 || offset > pC->payloadSize ){
+      if( aOffset[0] > 98307 || aOffset[0] > pC->payloadSize ){
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
@@ -2469,9 +2467,9 @@ case OP_Column: {
       ** (ex: pC->nHdrParsed<=p2) in the next section, we achieve a
       ** measurable performance gain.
       **
-      ** This branch is taken even if offset==0.  Such a record is never
+      ** This branch is taken even if aOffset[0]==0.  Such a record is never
       ** generated by SQLite, and could be considered corruption, but we
-      ** accept it for historical reasons.  When offset==0, the code this
+      ** accept it for historical reasons.  When aOffset[0]==0, the code this
       ** branch jumps to reads past the end of the record, but never more
       ** than a few bytes.  Even if the record occurs at the end of the page
       ** content area, the "page header" comes after the page content and so
@@ -2480,7 +2478,7 @@ case OP_Column: {
       */
       zData = pC->aRow;
       assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
-      testcase( offset==0 );
+      testcase( aOffset[0]==0 );
       goto op_column_read_header;
     }
   }

From fe0cf7a18c665b991a87e8329788563b750f9d9d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 16 Aug 2017 19:20:20 +0000
Subject: [PATCH 014/270] Avoid a test for CURTYPE_BTREE in
 sqlite3VdbeCursorMoveto() in order to reduce the size and improve the
 performance of OP_Column.

FossilOrigin-Name: f078deb25149b7b1881b7f3374b343d0677e82336d8fdd7f1cdd06d926b5dd57
---
 manifest       | 22 +++++++++++-----------
 manifest.uuid  |  2 +-
 src/btree.c    | 11 +++++++++++
 src/btree.h    |  1 +
 src/btreeInt.h | 10 +++++-----
 src/vdbe.c     | 13 ++++++++++---
 src/vdbeInt.h  | 12 ++++++------
 src/vdbeaux.c  | 23 +++++++++++------------
 8 files changed, 56 insertions(+), 38 deletions(-)

diff --git a/manifest b/manifest
index 514232a7c7..5543cf3302 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\slocal\svariable\sfrom\sOP_Column,\sfor\sa\ssmall\ssize\nreduction\sand\sperformance\sincrease.
-D 2017-08-16T14:16:19.879
+C Avoid\sa\stest\sfor\sCURTYPE_BTREE\sin\ssqlite3VdbeCursorMoveto()\sin\sorder\sto\sreduce\nthe\ssize\sand\simprove\sthe\sperformance\sof\sOP_Column.
+D 2017-08-16T19:20:20.392
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -397,9 +397,9 @@ F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c d2f5f347e56f8b7ed1bb798087045a3b98cd63f45fde7675fd24b9e88b61304d
-F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca
-F src/btreeInt.h f78671f594dafd88cf9a81253da04db81272b382d2dc074bb983d348b95d9d2d
+F src/btree.c e1a98bef27d6189df5c5b8b3c9b5e53d6b74bf9a936af97757c21e9ea6a5c6b2
+F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
+F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
 F src/callback.c 930648a084a3adc741c6471adfbdc50ba47ba3542421cb80a26f259f467de65e
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
@@ -522,11 +522,11 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c c8e7bec8764b26b8e80fadffab2b9c18e8ed5ab2e92d1be5a7aebeed18ca8be5
+F src/vdbe.c 711a1b2c0a6046483ebf4ac43c4c0ea5bef3e50534a4ab9e1d7e3542635cd009
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
-F src/vdbeInt.h ff2b7db0968d20e6184aee256d2e535d565f5a172e3588a78adb166a41fc4911
+F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
-F src/vdbeaux.c 1bef372f59f9e1dba5ead70cc5c24bf978bab0b9fdc2f69692afaa3a2d4dd8f3
+F src/vdbeaux.c 1f15018ef7abe22669967f70b02bfe6709be403f126f713dabb091b9d631859a
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c b7fac20534c79b7554dab2e8a180c585a8bc1b9c85149d1b2d9746cf314d06ed
 F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e3
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dc98a92f32511ee322b0207bd286e967248a8e59b418f11168eb31e34b0fa0fa
-R 8dc9658e95fe266a68d077704d4afe1a
+P 39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
+R 6446f3c323d2c08f3899473bfc8c1574
 U drh
-Z 4fdaa02107e5a271b3b6d501790c7a7c
+Z 68326b6d5ed2188a1ee820e19559d5ac
diff --git a/manifest.uuid b/manifest.uuid
index e7180c4267..eb2dc719d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
\ No newline at end of file
+f078deb25149b7b1881b7f3374b343d0677e82336d8fdd7f1cdd06d926b5dd57
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 50981b0575..ecdbb6d0a3 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -838,6 +838,17 @@ int sqlite3BtreeCursorHasMoved(BtCursor *pCur){
   return pCur->eState!=CURSOR_VALID;
 }
 
+/*
+** Return a pointer to a fake BtCursor object that will always answer
+** false to the sqlite3BtreeCursorHasMoved() routine above.  The fake
+** cursor returned must not be used with any other Btree interface.
+*/
+BtCursor *sqlite3BtreeFakeValidCursor(void){
+  static u8 fakeCursor = CURSOR_VALID;
+  assert( offsetof(BtCursor, eState)==0 );
+  return (BtCursor*)&fakeCursor;
+}
+
 /*
 ** This routine restores a cursor back to its original position after it
 ** has been moved by some outside activity (such as a btree rebalance or
diff --git a/src/btree.h b/src/btree.h
index b56eb85e68..e2c271cdc5 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -230,6 +230,7 @@ int sqlite3BtreeCursor(
   struct KeyInfo*,                     /* First argument to compare function */
   BtCursor *pCursor                    /* Space to write cursor structure */
 );
+BtCursor *sqlite3BtreeFakeValidCursor(void);
 int sqlite3BtreeCursorSize(void);
 void sqlite3BtreeCursorZero(BtCursor*);
 void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned);
diff --git a/src/btreeInt.h b/src/btreeInt.h
index f4e5f45fec..ac7a3c0a26 100644
--- a/src/btreeInt.h
+++ b/src/btreeInt.h
@@ -499,6 +499,11 @@ struct CellInfo {
 **    eState==FAULT:                   Cursor fault with skipNext as error code.
 */
 struct BtCursor {
+  u8 eState;                /* One of the CURSOR_XXX constants (see below) */
+  u8 curFlags;              /* zero or more BTCF_* flags defined below */
+  u8 curPagerFlags;         /* Flags to send to sqlite3PagerGet() */
+  u8 hints;                 /* As configured by CursorSetHints() */
+  int nOvflAlloc;           /* Allocated size of aOverflow[] array */
   Btree *pBtree;            /* The Btree to which this cursor belongs */
   BtShared *pBt;            /* The BtShared this cursor points to */
   BtCursor *pNext;          /* Forms a linked list of all cursors */
@@ -507,13 +512,8 @@ struct BtCursor {
   i64 nKey;                 /* Size of pKey, or last integer key */
   void *pKey;               /* Saved key that was cursor last known position */
   Pgno pgnoRoot;            /* The root page of this tree */
-  int nOvflAlloc;           /* Allocated size of aOverflow[] array */
   int skipNext;    /* Prev() is noop if negative. Next() is noop if positive.
                    ** Error code if eState==CURSOR_FAULT */
-  u8 curFlags;              /* zero or more BTCF_* flags defined below */
-  u8 curPagerFlags;         /* Flags to send to sqlite3PagerGet() */
-  u8 eState;                /* One of the CURSOR_XXX constants (see below) */
-  u8 hints;                 /* As configured by CursorSetHints() */
   /* All fields above are zeroed when the cursor is allocated.  See
   ** sqlite3BtreeCursorZero().  Fields that follow must be manually
   ** initialized. */
diff --git a/src/vdbe.c b/src/vdbe.c
index a4665ce79c..3f5d967d2d 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2413,8 +2413,10 @@ case OP_Column: {
   if( pC->cacheStatus!=p->cacheCtr ){                /*OPTIMIZATION-IF-FALSE*/
     if( pC->nullRow ){
       if( pC->eCurType==CURTYPE_PSEUDO ){
-        assert( pC->uc.pseudoTableReg>0 );
-        pReg = &aMem[pC->uc.pseudoTableReg];
+        /* For the special case of as pseudo-cursor, the seekResult field
+        ** identifies the register that holds the record */
+        assert( pC->seekResult>0 );
+        pReg = &aMem[pC->seekResult];
         assert( pReg->flags & MEM_Blob );
         assert( memIsValid(pReg) );
         pC->payloadSize = pC->szRow = pReg->n;
@@ -3628,8 +3630,13 @@ case OP_OpenPseudo: {
   pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO);
   if( pCx==0 ) goto no_mem;
   pCx->nullRow = 1;
-  pCx->uc.pseudoTableReg = pOp->p2;
+  pCx->seekResult = pOp->p2;
   pCx->isTable = 1;
+  /* Give this pseudo-cursor a fake BtCursor pointer so that pCx
+  ** can be safely passed to sqlite3VdbeCursorMoveto().  This avoids a test
+  ** for pCx->eCurType==CURTYPE_BTREE inside of sqlite3VdbeCursorMoveto()
+  ** which is a performance optimization */
+  pCx->uc.pCursor = sqlite3BtreeFakeValidCursor();
   assert( pOp->p5==0 );
   break;
 }
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index d8e47be500..cb783653c0 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -96,18 +96,18 @@ struct VdbeCursor {
   u32 cacheStatus;        /* Cache is valid if this matches Vdbe.cacheCtr */
   int seekResult;         /* Result of previous sqlite3BtreeMoveto() or 0
                           ** if there have been no prior seeks on the cursor. */
-  /* NB: seekResult does not distinguish between "no seeks have ever occurred
-  ** on this cursor" and "the most recent seek was an exact match". */
+  /* seekResult does not distinguish between "no seeks have ever occurred
+  ** on this cursor" and "the most recent seek was an exact match".
+  ** For CURTYPE_PSEUDO, seekResult is the register holding the record */
 
   /* When a new VdbeCursor is allocated, only the fields above are zeroed.
   ** The fields that follow are uninitialized, and must be individually
   ** initialized prior to first use. */
   VdbeCursor *pAltCursor; /* Associated index cursor from which to read */
   union {
-    BtCursor *pCursor;          /* CURTYPE_BTREE.  Btree cursor */
-    sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB.   Vtab cursor */
-    int pseudoTableReg;         /* CURTYPE_PSEUDO. Reg holding content. */
-    VdbeSorter *pSorter;        /* CURTYPE_SORTER. Sorter object */
+    BtCursor *pCursor;          /* CURTYPE_BTREE or _PSEUDO.  Btree cursor */
+    sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB.              Vtab cursor */
+    VdbeSorter *pSorter;        /* CURTYPE_SORTER.            Sorter object */
   } uc;
   KeyInfo *pKeyInfo;      /* Info about index keys needed by index cursors */
   u32 iHdrOffset;         /* Offset to next unparsed byte of the header */
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 226b2152ed..7f6632bfa4 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3139,19 +3139,18 @@ int sqlite3VdbeCursorRestore(VdbeCursor *p){
 */
 int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){
   VdbeCursor *p = *pp;
-  if( p->eCurType==CURTYPE_BTREE ){
-    if( p->deferredMoveto ){
-      int iMap;
-      if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){
-        *pp = p->pAltCursor;
-        *piCol = iMap - 1;
-        return SQLITE_OK;
-      }
-      return handleDeferredMoveto(p);
-    }
-    if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){
-      return handleMovedCursor(p);
+  assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO );
+  if( p->deferredMoveto ){
+    int iMap;
+    if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){
+      *pp = p->pAltCursor;
+      *piCol = iMap - 1;
+      return SQLITE_OK;
     }
+    return handleDeferredMoveto(p);
+  }
+  if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){
+    return handleMovedCursor(p);
   }
   return SQLITE_OK;
 }

From dc6b41ed4791d3ccfb2a311df03d32b09220a690 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 02:26:35 +0000
Subject: [PATCH 015/270] Defer schema resets when the query planner is
 running. Proposed fix for ticket [be436a7f4587ce517].

FossilOrigin-Name: a7bc7752ba0266684f5317e424a4ee9add4af002272082183519e708ab9ffc79
---
 manifest        | 18 +++++++++---------
 manifest.uuid   |  2 +-
 src/build.c     | 41 +++++++++++++++++++++++------------------
 src/callback.c  |  2 +-
 src/prepare.c   |  3 ++-
 src/sqliteInt.h |  2 ++
 6 files changed, 38 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index 5543cf3302..e83548519d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sa\stest\sfor\sCURTYPE_BTREE\sin\ssqlite3VdbeCursorMoveto()\sin\sorder\sto\sreduce\nthe\ssize\sand\simprove\sthe\sperformance\sof\sOP_Column.
-D 2017-08-16T19:20:20.392
+C Defer\sschema\sresets\swhen\sthe\squery\splanner\sis\srunning.\nProposed\sfix\sfor\sticket\s[be436a7f4587ce517].
+D 2017-08-17T02:26:35.841
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -400,8 +400,8 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c e1a98bef27d6189df5c5b8b3c9b5e53d6b74bf9a936af97757c21e9ea6a5c6b2
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c f8a85d2ad14c2201e10c2fe7185e1cb3f1db8b0c90d40d1fb99c20b135ab71bc
-F src/callback.c 930648a084a3adc741c6471adfbdc50ba47ba3542421cb80a26f259f467de65e
+F src/build.c 559bce114d59bb6dd795a7985a9eaac781d374ff31422d134dc147f9667a4d21
+F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
@@ -449,7 +449,7 @@ F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
 F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c 3cbb99757d7295997674972f9dd2331c5c544368854ca08954c9beb1e9b6145a
+F src/prepare.c 5da8a0563eff02e23584ec32863273eb3b1ac30bc736e1084efbcbf7379d1a56
 F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
@@ -460,7 +460,7 @@ F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c17
 F src/sqlite.h.in f0bd1abf5e27bd22b3bcaae2a861c1efc4ab7e752bf7eb102355135617eb8199
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 854a122ff0ebde410a66d4f967e9923de9002f73965c6c9fa0db544bf7e657d1
+F src/sqliteInt.h 6dddca4e215f4088aeaf60aebaa6d913397d61422733e160f25ab9dc53605a36
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1647,7 +1647,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
-R 6446f3c323d2c08f3899473bfc8c1574
+P f078deb25149b7b1881b7f3374b343d0677e82336d8fdd7f1cdd06d926b5dd57
+R 1aca6a5cd400a792803d04295cde6ca8
 U drh
-Z 68326b6d5ed2188a1ee820e19559d5ac
+Z ee696b9e27abcfab805d4537aa8e005a
diff --git a/manifest.uuid b/manifest.uuid
index eb2dc719d5..3c1067b3fc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f078deb25149b7b1881b7f3374b343d0677e82336d8fdd7f1cdd06d926b5dd57
\ No newline at end of file
+a7bc7752ba0266684f5317e424a4ee9add4af002272082183519e708ab9ffc79
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index cb3172e076..d64eac009e 100644
--- a/src/build.c
+++ b/src/build.c
@@ -514,28 +514,26 @@ void sqlite3CollapseDatabaseArray(sqlite3 *db){
 
 /*
 ** Reset the schema for the database at index iDb.  Also reset the
-** TEMP schema.
+** TEMP schema.  The reset is deferred if db->nSchemaLock is not zero.
+** Deferred resets may be run by calling with iDb<0.
 */
 void sqlite3ResetOneSchema(sqlite3 *db, int iDb){
-  Db *pDb;
+  int i;
   assert( iDb<db->nDb );
 
-  /* Case 1:  Reset the single schema identified by iDb */
-  pDb = &db->aDb[iDb];
-  assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
-  assert( pDb->pSchema!=0 );
-  sqlite3SchemaClear(pDb->pSchema);
-
-  /* If any database other than TEMP is reset, then also reset TEMP
-  ** since TEMP might be holding triggers that reference tables in the
-  ** other database.
-  */
-  if( iDb!=1 ){
-    pDb = &db->aDb[1];
-    assert( pDb->pSchema!=0 );
-    sqlite3SchemaClear(pDb->pSchema);
+  if( iDb>=0 ){
+    assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
+    DbSetProperty(db, iDb, DB_ResetWanted);
+    DbSetProperty(db, 1, DB_ResetWanted);
+  }
+
+  if( db->nSchemaLock==0 ){
+    for(i=0; i<db->nDb; i++){
+      if( DbHasProperty(db, i, DB_ResetWanted) ){
+        sqlite3SchemaClear(db->aDb[i].pSchema);
+      }
+    }
   }
-  return;
 }
 
 /*
@@ -545,6 +543,7 @@ void sqlite3ResetOneSchema(sqlite3 *db, int iDb){
 void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
   int i;
   sqlite3BtreeEnterAll(db);
+  assert( db->nSchemaLock==0 );
   for(i=0; i<db->nDb; i++){
     Db *pDb = &db->aDb[i];
     if( pDb->pSchema ){
@@ -2155,6 +2154,9 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
   int nErr = 0;     /* Number of errors encountered */
   int n;            /* Temporarily holds the number of cursors assigned */
   sqlite3 *db = pParse->db;  /* Database connection for malloc errors */
+#ifndef SQLITE_OMIT_VIRTUALTABLE	
+  int rc;
+#endif
 #ifndef SQLITE_OMIT_AUTHORIZATION
   sqlite3_xauth xAuth;       /* Saved xAuth pointer */
 #endif
@@ -2162,7 +2164,10 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
   assert( pTable );
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  if( sqlite3VtabCallConnect(pParse, pTable) ){
+  db->nSchemaLock++;
+  rc = sqlite3VtabCallConnect(pParse, pTable);
+  db->nSchemaLock--;
+  if( rc ){
     return SQLITE_ERROR;
   }
   if( IsVirtual(pTable) ) return 0;
diff --git a/src/callback.c b/src/callback.c
index 10505414c1..e08924b2c2 100644
--- a/src/callback.c
+++ b/src/callback.c
@@ -457,8 +457,8 @@ void sqlite3SchemaClear(void *p){
   pSchema->pSeqTab = 0;
   if( pSchema->schemaFlags & DB_SchemaLoaded ){
     pSchema->iGeneration++;
-    pSchema->schemaFlags &= ~DB_SchemaLoaded;
   }
+  pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted);
 }
 
 /*
diff --git a/src/prepare.c b/src/prepare.c
index 4fa59e5bef..c31b810820 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -686,7 +686,8 @@ static int sqlite3LockAndPrepare(
   sqlite3_mutex_enter(db->mutex);
   sqlite3BtreeEnterAll(db);
   rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);
-  if( rc==SQLITE_SCHEMA ){
+  if( rc==SQLITE_SCHEMA && db->nSchemaLock==0 ){
+    sqlite3ResetOneSchema(db, -1);
     sqlite3_finalize(*ppStmt);
     rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);
   }
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c457b5401b..8aa9832e8a 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1213,6 +1213,7 @@ struct Schema {
 #define DB_SchemaLoaded    0x0001  /* The schema has been loaded */
 #define DB_UnresetViews    0x0002  /* Some views have defined column names */
 #define DB_Empty           0x0004  /* The file is empty (length 0 bytes) */
+#define DB_ResetWanted     0x0008  /* Reset the schema when nSchemaLock==0 */
 
 /*
 ** The number of different kinds of things that can be limited
@@ -1329,6 +1330,7 @@ struct sqlite3 {
   u32 flags;                    /* flags settable by pragmas. See below */
   i64 lastRowid;                /* ROWID of most recent insert (see above) */
   i64 szMmap;                   /* Default mmap_size setting */
+  u32 nSchemaLock;              /* Do not reset the schema when non-zero */
   unsigned int openFlags;       /* Flags passed to sqlite3_vfs.xOpen() */
   int errCode;                  /* Most recent error code (SQLITE_*) */
   int errMask;                  /* & result codes with this before returning */

From 865c3c58ab14ff1b520e709a49232f9d48e79aea Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 17 Aug 2017 14:12:16 +0000
Subject: [PATCH 016/270] Add test cases for ticket [be436a7f4587ce517] using
 virtual table modules fts5 and rtree.

FossilOrigin-Name: 2101b4208787d297788e05f2bd82d4a9aff26e2237a7016ac857a52fb5252ce0
---
 ext/fts5/test/fts5connect.test | 247 +++++++++++++++++++++++++++++++++
 ext/rtree/rtreeconnect.test    |  56 ++++++++
 manifest                       |  14 +-
 manifest.uuid                  |   2 +-
 4 files changed, 312 insertions(+), 7 deletions(-)
 create mode 100644 ext/fts5/test/fts5connect.test
 create mode 100644 ext/rtree/rtreeconnect.test

diff --git a/ext/fts5/test/fts5connect.test b/ext/fts5/test/fts5connect.test
new file mode 100644
index 0000000000..c615d4c734
--- /dev/null
+++ b/ext/fts5/test/fts5connect.test
@@ -0,0 +1,247 @@
+# 2017 August 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5connect
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# The tests in this file test the outcome of a schema-reset happening 
+# within the xConnect() method of an FTS5 table. At one point this
+# was causing a problem in SQLite. Each test proceeds as follows:
+#
+#   1. Connection [db] opens the db and reads from some unrelated, non-FTS5
+#      table causing SQLite to load the db schema into memory.
+#
+#   2. Connection [db2] opens the db and modifies the db schema.
+#
+#   3. Connection [db] reads or writes an existing fts5 table. That the
+#      schema has been modified is detected inside the fts5 xConnect() 
+#      callback that is invoked by sqlite3_prepare(). 
+#
+#   4. Verify that the statement in 3 has worked. SQLite should detect
+#      that the schema has changed and successfully prepare the 
+#      statement against the new schema.
+#
+# Test plan:
+#
+#   1.*: Trigger the xConnect()/schema-reset using statements executed
+#        directly against an FTS5 table.
+#
+#   2.*: Using various statements executed by various BEFORE triggers.
+#
+#   3.*: Using various statements executed by various AFTER triggers.
+#
+#   4.*: Using various statements executed by various INSTEAD OF triggers.
+#
+
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, b);
+  CREATE TABLE abc(x INTEGER PRIMARY KEY);
+  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
+
+  INSERT INTO ft1 VALUES('one', 'two');
+  INSERT INTO ft1 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft1" {one two three four}
+  2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft1" {five six three four}
+  4 "INSERT INTO ft1 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft1" {five six three four seven eight}
+  6 "DELETE FROM ft1 WHERE rowid=2" {}
+  7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft1" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 1.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 1.$tn.2 $sql $res
+
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO ft1(ft1) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(a, b);
+  CREATE TABLE t2(a, b);
+  CREATE TABLE log(txt);
+
+  CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN
+    INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN
+    DELETE FROM ft2 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN
+    UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t2 VALUES('one', 'two');
+  INSERT INTO t2 VALUES('three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t2" {one two three four}
+  2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft2" {five six three four}
+  4 "INSERT INTO t2 VALUES('seven', 'eight')" {}
+  5 "SELECT * FROM ft2" {five six three four seven eight}
+  6 "DELETE FROM t2 WHERE rowid=2" {}
+  7 "UPDATE t2 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft2" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 2.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 2.$tn.2 $sql $res
+
+  do_execsql_test 2.$tn.3 {
+    INSERT INTO ft2(ft2) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(a, b);
+  CREATE TABLE t3(a, b);
+
+  CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN
+    INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN
+    DELETE FROM ft3 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN
+    UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM t3" {one two three four}
+  2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft3" {five six three four}
+  4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft3" {five six three four seven eight}
+  6 "DELETE FROM t3 WHERE rowid=2" {}
+  7 "UPDATE t3 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft3" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 3.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 3.$tn.2 $sql $res
+
+  do_execsql_test 3.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft4 USING fts5(a, b);
+  CREATE VIEW v4 AS SELECT rowid, * FROM ft4;
+
+  CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN
+    INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b);
+    INSERT INTO log VALUES('insert');
+  END;
+
+  CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN
+    DELETE FROM ft4 WHERE rowid = old.rowid;
+    INSERT INTO log VALUES('delete');
+  END;
+
+  CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN
+    UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid;
+    INSERT INTO log VALUES('update');
+  END;
+
+  INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two');
+  INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four');
+}
+
+foreach {tn sql res} {
+  1 "SELECT * FROM ft4" {one two three four}
+  2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {}
+  3 "SELECT * FROM ft4" {five six three four}
+  4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
+  5 "SELECT * FROM ft4" {five six three four seven eight}
+  6 "DELETE FROM v4 WHERE rowid=2" {}
+  7 "UPDATE v4 SET b='nine' WHERE rowid=1" {}
+  8 "SELECT * FROM ft4" {five nine seven eight}
+} {
+
+  catch { db close }
+  catch { db2 close }
+  sqlite3 db  test.db
+  sqlite3 db2 test.db
+
+  do_test 4.$tn.1 {
+    db eval { INSERT INTO abc DEFAULT VALUES }
+    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
+  } {}
+
+  do_execsql_test 4.$tn.2 $sql $res
+
+  do_execsql_test 4.$tn.3 {
+    INSERT INTO ft3(ft3) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
diff --git a/ext/rtree/rtreeconnect.test b/ext/rtree/rtreeconnect.test
new file mode 100644
index 0000000000..16d04d9a04
--- /dev/null
+++ b/ext/rtree/rtreeconnect.test
@@ -0,0 +1,56 @@
+# 2017 August 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the r-tree extension. Specifically,
+# the impact of an SQLITE_SCHEMA error within the rtree module xConnect
+# callback.
+#
+
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+set testprefix rtreeconnect
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE r1 USING rtree(id, x1, x2, y1, y2);
+  CREATE TABLE t1(id, x1, x2, y1, y2);
+  CREATE TABLE log(l);
+
+  CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+    INSERT INTO r1 VALUES(new.id, new.x1, new.x2, new.y1, new.y2);
+    INSERT INTO log VALUES('r1: ' || new.id);
+  END;
+}
+
+db close
+sqlite3 db  test.db
+sqlite3 db2 test.db
+
+do_test 1.1 {
+  db eval { INSERT INTO log VALUES('startup'); }
+  db2 eval { CREATE TABLE newtable(x,y); }
+} {}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1 VALUES(1, 2, 3, 4, 5);
+}
+
+db2 close
+db close
+
+finish_test
diff --git a/manifest b/manifest
index e83548519d..0c8553907a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Defer\sschema\sresets\swhen\sthe\squery\splanner\sis\srunning.\nProposed\sfix\sfor\sticket\s[be436a7f4587ce517].
-D 2017-08-17T02:26:35.841
+C Add\stest\scases\sfor\sticket\s[be436a7f4587ce517]\susing\svirtual\stable\smodules\sfts5\nand\srtree.
+D 2017-08-17T14:12:16.980
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -140,6 +140,7 @@ F ext/fts5/test/fts5colset.test a30473451321bbf0b6218af62e96b4ae5fa99931cfdb210b
 F ext/fts5/test/fts5columnsize.test 45459ce4dd9fd853b6044cdc9674921bff89e3d840f348ca8c1630f9edbf5482
 F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825ce437d26afe0817f
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
+F ext/fts5/test/fts5connect.test b12a2a8b02af3c31c18abbc33aa8100d364de19a888a44457484d21dbccb18a7
 F ext/fts5/test/fts5content.test 688d5ac7af194ebc67495daea76a69e3cd5480122c2320e72d41241b423b4116
 F ext/fts5/test/fts5corrupt.test 8957f0f7e57e0f8a102c5b6e1a7326d6a1966b28e1d99c5883822af1e6038e9e
 F ext/fts5/test/fts5corrupt2.test 6deaf9f9606b3c957529db9881622bb3a7829b19bb3cdf8f276f074d684ede56
@@ -343,6 +344,7 @@ F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4
 F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea
+F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d
 F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28
 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
@@ -1647,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f078deb25149b7b1881b7f3374b343d0677e82336d8fdd7f1cdd06d926b5dd57
-R 1aca6a5cd400a792803d04295cde6ca8
-U drh
-Z ee696b9e27abcfab805d4537aa8e005a
+P a7bc7752ba0266684f5317e424a4ee9add4af002272082183519e708ab9ffc79
+R c613de5ae36b19d57fec37de442df285
+U dan
+Z ec0c29a7c81c6674b289e2e6cbe8b9e8
diff --git a/manifest.uuid b/manifest.uuid
index 3c1067b3fc..fd647ce99e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7bc7752ba0266684f5317e424a4ee9add4af002272082183519e708ab9ffc79
\ No newline at end of file
+2101b4208787d297788e05f2bd82d4a9aff26e2237a7016ac857a52fb5252ce0
\ No newline at end of file

From 967f8f9e0717de8c8127755a3ca770d8b26858dc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 14:47:56 +0000
Subject: [PATCH 017/270] Remove an unnecessary branch from the
 [be436a7f4587ce517ddc] fix.

FossilOrigin-Name: fb6ca99b88c67414eab425e2b4ddb6a5e2fd48a6b2cd96a34627c86a5b568c9b
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/prepare.c |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 0c8553907a..427ca49809 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\scases\sfor\sticket\s[be436a7f4587ce517]\susing\svirtual\stable\smodules\sfts5\nand\srtree.
-D 2017-08-17T14:12:16.980
+C Remove\san\sunnecessary\sbranch\sfrom\sthe\s[be436a7f4587ce517ddc]\sfix.
+D 2017-08-17T14:47:56.368
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -451,7 +451,7 @@ F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
 F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c 5da8a0563eff02e23584ec32863273eb3b1ac30bc736e1084efbcbf7379d1a56
+F src/prepare.c 9e880c0efb5d7f9101bb34c0a87daf6e1e5284c34024fdb811e67bb02fdd299b
 F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7bc7752ba0266684f5317e424a4ee9add4af002272082183519e708ab9ffc79
-R c613de5ae36b19d57fec37de442df285
-U dan
-Z ec0c29a7c81c6674b289e2e6cbe8b9e8
+P 2101b4208787d297788e05f2bd82d4a9aff26e2237a7016ac857a52fb5252ce0
+R a88a9175d0467a8bcee3d90b3070177c
+U drh
+Z f282ee329c3784824fa5652eca8490b4
diff --git a/manifest.uuid b/manifest.uuid
index fd647ce99e..9e80a745e0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2101b4208787d297788e05f2bd82d4a9aff26e2237a7016ac857a52fb5252ce0
\ No newline at end of file
+fb6ca99b88c67414eab425e2b4ddb6a5e2fd48a6b2cd96a34627c86a5b568c9b
\ No newline at end of file
diff --git a/src/prepare.c b/src/prepare.c
index c31b810820..9f0704c947 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -686,7 +686,7 @@ static int sqlite3LockAndPrepare(
   sqlite3_mutex_enter(db->mutex);
   sqlite3BtreeEnterAll(db);
   rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);
-  if( rc==SQLITE_SCHEMA && db->nSchemaLock==0 ){
+  if( rc==SQLITE_SCHEMA ){
     sqlite3ResetOneSchema(db, -1);
     sqlite3_finalize(*ppStmt);
     rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);

From 6362bbe68d26560660e017c0ed0c420f99fdb052 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 18:17:24 +0000
Subject: [PATCH 018/270] The RTREE extension should return
 SQLITE_CORRUPT_VTAB, not just SQLITE_CORRUPT when it encounters incorrectly
 formatted shadow tables.

FossilOrigin-Name: 0712f057ef3dcd907984dda30f6d961a29b61c1d2b25627028c4e227ec85dbba
---
 ext/rtree/rtree.c     |  2 +-
 ext/rtree/rtreeA.test |  2 +-
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index d97cb45491..40a8676d7b 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -3414,7 +3414,7 @@ static int getNodeSize(
     if( rc!=SQLITE_OK ){
       *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
     }else if( pRtree->iNodeSize<(512-64) ){
-      rc = SQLITE_CORRUPT;
+      rc = SQLITE_CORRUPT_VTAB;
       *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"",
                                pRtree->zName);
     }
diff --git a/ext/rtree/rtreeA.test b/ext/rtree/rtreeA.test
index 0a718a6f92..d583a15264 100644
--- a/ext/rtree/rtreeA.test
+++ b/ext/rtree/rtreeA.test
@@ -230,7 +230,7 @@ do_catchsql_test rtreeA-7.110 {
 } {1 {undersize RTree blobs in "t1_node"}}
 do_test rtreeA-7.120 {
   sqlite3_extended_errcode db
-} {SQLITE_CORRUPT}
+} {SQLITE_CORRUPT_VTAB}
 
 
 
diff --git a/manifest b/manifest
index 427ca49809..b2ab03eb02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\sbranch\sfrom\sthe\s[be436a7f4587ce517ddc]\sfix.
-D 2017-08-17T14:47:56.368
+C The\sRTREE\sextension\sshould\sreturn\sSQLITE_CORRUPT_VTAB,\snot\sjust\sSQLITE_CORRUPT\nwhen\sit\sencounters\sincorrectly\sformatted\sshadow\stables.
+D 2017-08-17T18:17:24.544
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -324,7 +324,7 @@ F ext/rbu/sqlite3rbu.c 920941a6ff7dbbea0970717c43662878fda5c37e43752de329f0fdd76
 F ext/rbu/sqlite3rbu.h 82c102e5ae41025e3b245d3d5944315f82811da85e2cd363a75caa97cbd0cd3e
 F ext/rbu/test_rbu.c ec18cfc69a104309df23c359e3c80306c9a6bdd1d2c53c8b70ae158e9832dcd6
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 4f1804b80ae06ddf7ff69192aacdceee283646dc6a328acb951f116147445212
+F ext/rtree/rtree.c cf84d52958a7ec6a506f1711e119db847ed6bb5dedde78a58e97503287afcda1
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 4fdd60ae034e43f2fefc26492032d02e742e8b14d468b7c51d95a1e2fa47cf00
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
@@ -335,7 +335,7 @@ F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196
 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971
 F ext/rtree/rtree8.test 076d9d5b783b61b7a23a5ab45fc899551dfffd821974f36ee599ff29f4de7a61
 F ext/rtree/rtree9.test 8bfa84dfaba1c897468a2448c28db0a00ad12d464225b5993c7814e907f3776f
-F ext/rtree/rtreeA.test abb1e2434defc8cdc5e3195a18ead3681cae04565c06069251d1998796e77d55
+F ext/rtree/rtreeA.test c09ad3f76c08feac00770685ff50ca12966dc0c641bf19a982b26a80643b46d1
 F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e
 F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d
 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2101b4208787d297788e05f2bd82d4a9aff26e2237a7016ac857a52fb5252ce0
-R a88a9175d0467a8bcee3d90b3070177c
+P fb6ca99b88c67414eab425e2b4ddb6a5e2fd48a6b2cd96a34627c86a5b568c9b
+R 44adf3aab86d2f98e7899082e69a019f
 U drh
-Z f282ee329c3784824fa5652eca8490b4
+Z f778c72442b5a965c1d6d29d9b5a7947
diff --git a/manifest.uuid b/manifest.uuid
index 9e80a745e0..01e27acd7c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fb6ca99b88c67414eab425e2b4ddb6a5e2fd48a6b2cd96a34627c86a5b568c9b
\ No newline at end of file
+0712f057ef3dcd907984dda30f6d961a29b61c1d2b25627028c4e227ec85dbba
\ No newline at end of file

From efaffb64fdbc2e00701ac73d3e5481e8f6d04724 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 18:23:46 +0000
Subject: [PATCH 019/270] In sqlite3ViewGetColumnNames(), return the number of
 errors, not an error code.

FossilOrigin-Name: f13682ea2350ba366026a4a58e59591af378c496b92da60803c4c642c3bfc8de
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/build.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b2ab03eb02..0dd29b18ac 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sRTREE\sextension\sshould\sreturn\sSQLITE_CORRUPT_VTAB,\snot\sjust\sSQLITE_CORRUPT\nwhen\sit\sencounters\sincorrectly\sformatted\sshadow\stables.
-D 2017-08-17T18:17:24.544
+C In\ssqlite3ViewGetColumnNames(),\sreturn\sthe\snumber\sof\serrors,\snot\san\serror\scode.
+D 2017-08-17T18:23:46.704
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -402,7 +402,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c e1a98bef27d6189df5c5b8b3c9b5e53d6b74bf9a936af97757c21e9ea6a5c6b2
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 559bce114d59bb6dd795a7985a9eaac781d374ff31422d134dc147f9667a4d21
+F src/build.c 5b81049a4cea3f547ddb4efc6f56345894524248816dc1ca1511b99be3f7d3ad
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb6ca99b88c67414eab425e2b4ddb6a5e2fd48a6b2cd96a34627c86a5b568c9b
-R 44adf3aab86d2f98e7899082e69a019f
+P 0712f057ef3dcd907984dda30f6d961a29b61c1d2b25627028c4e227ec85dbba
+R f75e24aaa696ff01027f9a4363031c06
 U drh
-Z f778c72442b5a965c1d6d29d9b5a7947
+Z 9049600ac1caae144c300dc50a09a71b
diff --git a/manifest.uuid b/manifest.uuid
index 01e27acd7c..2289dcd507 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0712f057ef3dcd907984dda30f6d961a29b61c1d2b25627028c4e227ec85dbba
\ No newline at end of file
+f13682ea2350ba366026a4a58e59591af378c496b92da60803c4c642c3bfc8de
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index d64eac009e..55407e77d9 100644
--- a/src/build.c
+++ b/src/build.c
@@ -2168,7 +2168,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
   rc = sqlite3VtabCallConnect(pParse, pTable);
   db->nSchemaLock--;
   if( rc ){
-    return SQLITE_ERROR;
+    return 1;
   }
   if( IsVirtual(pTable) ) return 0;
 #endif

From 6f7fbcf0bb90adbbf988bd2fbda7fa78a0baf89e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 18:54:27 +0000
Subject: [PATCH 020/270] Size optimization in the authorizer error message
 generation logic.

FossilOrigin-Name: 0367a4d58682a64d7ed4c5a4b4377899e22432851587c649d419efb6d7bac250
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/auth.c    |  8 +++-----
 3 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 0dd29b18ac..c2037b0fb1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\ssqlite3ViewGetColumnNames(),\sreturn\sthe\snumber\sof\serrors,\snot\san\serror\scode.
-D 2017-08-17T18:23:46.704
+C Size\soptimization\sin\sthe\sauthorizer\serror\smessage\sgeneration\slogic.
+D 2017-08-17T18:54:27.587
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -395,7 +395,7 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421
 F src/attach.c 07b706e336fd3cedbd855e1f8266d10e82fecae07daf86717b5760cd7784c584
-F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
+F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0712f057ef3dcd907984dda30f6d961a29b61c1d2b25627028c4e227ec85dbba
-R f75e24aaa696ff01027f9a4363031c06
+P f13682ea2350ba366026a4a58e59591af378c496b92da60803c4c642c3bfc8de
+R 40661ee0bd8f99ce6a0763a7096049ca
 U drh
-Z 9049600ac1caae144c300dc50a09a71b
+Z 4b024de8a532f345dd700cff7b7b4750
diff --git a/manifest.uuid b/manifest.uuid
index 2289dcd507..684f646ad4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f13682ea2350ba366026a4a58e59591af378c496b92da60803c4c642c3bfc8de
\ No newline at end of file
+0367a4d58682a64d7ed4c5a4b4377899e22432851587c649d419efb6d7bac250
\ No newline at end of file
diff --git a/src/auth.c b/src/auth.c
index dabc435b4a..7d6f851d89 100644
--- a/src/auth.c
+++ b/src/auth.c
@@ -118,11 +118,9 @@ int sqlite3AuthReadCol(
 #endif
                 );
   if( rc==SQLITE_DENY ){
-    if( db->nDb>2 || iDb!=0 ){
-      sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol);
-    }else{
-      sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol);
-    }
+    char *z = sqlite3_mprintf("%s.%s", zTab, zCol);
+    if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z);
+    sqlite3ErrorMsg(pParse, "access to %z is prohibited", z);
     pParse->rc = SQLITE_AUTH;
   }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){
     sqliteAuthBadReturnCode(pParse);

From dceed86d07146bedbe8b54a613601d2db0d937f8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 19:23:16 +0000
Subject: [PATCH 021/270] Small optimization in the Expr tree walker.

FossilOrigin-Name: 264a5e1b75ee073bd019483e289c3d8d79abcf5a765435be23ac3c21b1db8246
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/walker.c  | 26 +++++++++++++++-----------
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/manifest b/manifest
index c2037b0fb1..92d8e3f282 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\soptimization\sin\sthe\sauthorizer\serror\smessage\sgeneration\slogic.
-D 2017-08-17T18:54:27.587
+C Small\soptimization\sin\sthe\sExpr\stree\swalker.
+D 2017-08-17T19:23:16.310
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -537,7 +537,7 @@ F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
-F src/walker.c a7ca64ce08a83a20d32186fbe06bca9234e348cfcf07959ee322fdc3e8a6173a
+F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c cbe8ddffbcec7ce86f7a800fe8fd10aee412c76c87e0dd3732a1682e68d74cd9
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
 F src/wherecode.c e7be3b7f4c11908500cdf02b299d190d3742659533f58e0f4047962fdb5a48da
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f13682ea2350ba366026a4a58e59591af378c496b92da60803c4c642c3bfc8de
-R 40661ee0bd8f99ce6a0763a7096049ca
+P 0367a4d58682a64d7ed4c5a4b4377899e22432851587c649d419efb6d7bac250
+R ff1cbce154569f3d7cb9695eac015fa5
 U drh
-Z 4b024de8a532f345dd700cff7b7b4750
+Z 24e86d635456e5a8dccd5aa790210fdc
diff --git a/manifest.uuid b/manifest.uuid
index 684f646ad4..fb45173c1a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0367a4d58682a64d7ed4c5a4b4377899e22432851587c649d419efb6d7bac250
\ No newline at end of file
+264a5e1b75ee073bd019483e289c3d8d79abcf5a765435be23ac3c21b1db8246
\ No newline at end of file
diff --git a/src/walker.c b/src/walker.c
index 2e292295de..49f9fa1897 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -40,18 +40,22 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
   int rc;
   testcase( ExprHasProperty(pExpr, EP_TokenOnly) );
   testcase( ExprHasProperty(pExpr, EP_Reduced) );
-  rc = pWalker->xExprCallback(pWalker, pExpr);
-  if( rc ) return rc & WRC_Abort;
-  if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){
-    if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
-    assert( pExpr->x.pList==0 || pExpr->pRight==0 );
-    if( pExpr->pRight ){
-      if( walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
-    }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){
-      if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
-    }else if( pExpr->x.pList ){
-      if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
+  while(1){
+    rc = pWalker->xExprCallback(pWalker, pExpr);
+    if( rc ) return rc & WRC_Abort;
+    if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){
+      if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
+       assert( pExpr->x.pList==0 || pExpr->pRight==0 );
+      if( pExpr->pRight ){
+        pExpr = pExpr->pRight;
+        continue;
+      }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){
+        if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
+      }else if( pExpr->x.pList ){
+        if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
+      }
     }
+    break;
   }
   return WRC_Continue;
 }

From ceb4b1dbdde735e851b3bf0d8656cfbe3c29b11a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 17 Aug 2017 20:53:07 +0000
Subject: [PATCH 022/270] Use the __builtin_clzll() function of gcc to improve
 the performance and reduce the size of the sqlite3LogEst() routine.

FossilOrigin-Name: a42a438cbbd721765ca55e71c464552dbaa494050cf472593599b8c7f0249516
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/util.c    |  6 ++++++
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 92d8e3f282..bb8ae26a38 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\soptimization\sin\sthe\sExpr\stree\swalker.
-D 2017-08-17T19:23:16.310
+C Use\sthe\s__builtin_clzll()\sfunction\sof\sgcc\sto\simprove\sthe\sperformance\sand\nreduce\sthe\ssize\sof\sthe\ssqlite3LogEst()\sroutine.
+D 2017-08-17T20:53:07.912
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -522,7 +522,7 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
 F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
+F src/util.c 773843506ce694714bc96fe67c30c37015f90ef515d0e70f1f8d5c9c24088152
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 711a1b2c0a6046483ebf4ac43c4c0ea5bef3e50534a4ab9e1d7e3542635cd009
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0367a4d58682a64d7ed4c5a4b4377899e22432851587c649d419efb6d7bac250
-R ff1cbce154569f3d7cb9695eac015fa5
+P 264a5e1b75ee073bd019483e289c3d8d79abcf5a765435be23ac3c21b1db8246
+R 08353d9c0ca9dacc46ddab1c0e437979
 U drh
-Z 24e86d635456e5a8dccd5aa790210fdc
+Z 35417eeee14d21861c3f9ed9415c7596
diff --git a/manifest.uuid b/manifest.uuid
index fb45173c1a..04447e26d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-264a5e1b75ee073bd019483e289c3d8d79abcf5a765435be23ac3c21b1db8246
\ No newline at end of file
+a42a438cbbd721765ca55e71c464552dbaa494050cf472593599b8c7f0249516
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index 7c9c5f54b7..a6d349296f 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1413,8 +1413,14 @@ LogEst sqlite3LogEst(u64 x){
     if( x<2 ) return 0;
     while( x<8 ){  y -= 10; x <<= 1; }
   }else{
+#if GCC_VERSION>=5004000
+    int i = 60 - __builtin_clzll(x);
+    y += i*10;
+    x >>= i;
+#else
     while( x>255 ){ y += 40; x >>= 4; }  /*OPTIMIZATION-IF-TRUE*/
     while( x>15 ){  y += 10; x >>= 1; }
+#endif
   }
   return a[x&7] + y - 10;
 }

From 62f6f51ae142112c8354460fc70f5c3471d329c4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 18 Aug 2017 08:29:37 +0000
Subject: [PATCH 023/270] Only use indexes on expressions to optimize ORDER BY
 and GROUP BY if the collation sequence matches. Possible fix for [e20dd54a].

FossilOrigin-Name: 37e1900880b70be6802eaf43b0e568fda709a1dd6083d8be11e5a7a7d1fda41a
---
 manifest             | 16 ++++----
 manifest.uuid        |  2 +-
 src/where.c          |  2 +-
 test/indexexpr2.test | 97 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 107 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index bb8ae26a38..dad7e502b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sthe\s__builtin_clzll()\sfunction\sof\sgcc\sto\simprove\sthe\sperformance\sand\nreduce\sthe\ssize\sof\sthe\ssqlite3LogEst()\sroutine.
-D 2017-08-17T20:53:07.912
+C Only\suse\sindexes\son\sexpressions\sto\soptimize\sORDER\sBY\sand\sGROUP\sBY\sif\sthe\ncollation\ssequence\smatches.\sPossible\sfix\sfor\s[e20dd54a].
+D 2017-08-18T08:29:37.727
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -538,7 +538,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c cbe8ddffbcec7ce86f7a800fe8fd10aee412c76c87e0dd3732a1682e68d74cd9
+F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
 F src/wherecode.c e7be3b7f4c11908500cdf02b299d190d3742659533f58e0f4047962fdb5a48da
 F src/whereexpr.c fe1fe600d7334e91f3d9d487021362d543fba8ab2f1be5e0d68063d619379c05
@@ -943,7 +943,7 @@ F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
-F test/indexexpr2.test 3ddd7f23bc381b9f2b7a15f2d083b1a4078e7733dce8295602ecfa3c74a34cf9
+F test/indexexpr2.test 2237f1408efa921bd66d0a09ebf0208cb0c228c1bc3b3a18e9fb8fc87d6ed90b
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 264a5e1b75ee073bd019483e289c3d8d79abcf5a765435be23ac3c21b1db8246
-R 08353d9c0ca9dacc46ddab1c0e437979
-U drh
-Z 35417eeee14d21861c3f9ed9415c7596
+P a42a438cbbd721765ca55e71c464552dbaa494050cf472593599b8c7f0249516
+R 01b1b734d02df776d0a6e39431469431
+U dan
+Z 25c783941cfeb73ae08636afb42efa7c
diff --git a/manifest.uuid b/manifest.uuid
index 04447e26d5..eae0feb8f2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a42a438cbbd721765ca55e71c464552dbaa494050cf472593599b8c7f0249516
\ No newline at end of file
+37e1900880b70be6802eaf43b0e568fda709a1dd6083d8be11e5a7a7d1fda41a
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 0e82e471f4..5402977c30 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3689,7 +3689,7 @@ static i8 wherePathSatisfiesOrderBy(
               continue;
             }
           }
-          if( iColumn>=0 ){
+          if( iColumn!=XN_ROWID ){
             pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
             if( !pColl ) pColl = db->pDfltColl;
             if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
diff --git a/test/indexexpr2.test b/test/indexexpr2.test
index c72561f347..7fa226cf8f 100644
--- a/test/indexexpr2.test
+++ b/test/indexexpr2.test
@@ -40,4 +40,101 @@ do_execsql_test 2.1 {
   SELECT a+1, quote(a+1) FROM t1 ORDER BY 1;
 } {2 2 3 3 4 4}
 
+#-------------------------------------------------------------------------
+# At one point SQLite was incorrectly using indexes on expressions to
+# optimize ORDER BY and GROUP BY clauses even when the collation
+# sequences of the query and index did not match (ticket [e20dd54ab0e4]).
+# The following tests - 3.* - attempt to verify that this has been fixed.
+#
+
+reset_db
+do_execsql_test 3.1.0 {
+  CREATE TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+} {}
+
+do_eqp_test 3.1.1 {
+  SELECT b FROM t1 WHERE b IS NOT NULL AND a IS NULL 
+  GROUP BY b COLLATE nocase
+  ORDER BY b COLLATE nocase;
+} {
+  0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=? AND b>?)}
+  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+}
+
+do_execsql_test 3.2.0 {
+  CREATE TABLE t2(x);
+
+  INSERT INTO t2 VALUES('.ABC');
+  INSERT INTO t2 VALUES('.abcd');
+  INSERT INTO t2 VALUES('.defg');
+  INSERT INTO t2 VALUES('.DEF');
+} {}
+
+do_execsql_test 3.2.1 {
+  SELECT x FROM t2 ORDER BY substr(x, 2) COLLATE nocase;
+} {
+  .ABC .abcd .DEF .defg
+}
+
+do_execsql_test 3.2.2 {
+  CREATE INDEX i2 ON t2( substr(x, 2) );
+  SELECT x FROM t2 ORDER BY substr(x, 2) COLLATE nocase;
+} {
+  .ABC .abcd .DEF .defg
+}
+
+do_execsql_test 3.3.0 {
+  CREATE TABLE t3(x);
+}
+
+do_eqp_test 3.3.1 {
+  SELECT json_extract(x, '$.b') FROM t2 
+  WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
+  GROUP BY json_extract(x, '$.b') COLLATE nocase
+  ORDER BY json_extract(x, '$.b') COLLATE nocase;
+} {
+  0 0 0 {SCAN TABLE t2} 
+  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+}
+
+do_execsql_test 3.3.2 {
+  CREATE INDEX i3 ON t3(json_extract(x, '$.a'), json_extract(x, '$.b'));
+} {}
+
+do_eqp_test 3.3.3 {
+  SELECT json_extract(x, '$.b') FROM t3 
+  WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
+  GROUP BY json_extract(x, '$.b') COLLATE nocase
+  ORDER BY json_extract(x, '$.b') COLLATE nocase;
+} {
+  0 0 0 {SEARCH TABLE t3 USING INDEX i3 (<expr>=?)} 
+  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+}
+
+do_execsql_test 3.4.0 {
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES('.ABC', 1);
+  INSERT INTO t4 VALUES('.abc', 2);
+  INSERT INTO t4 VALUES('.ABC', 3);
+  INSERT INTO t4 VALUES('.abc', 4);
+}
+
+do_execsql_test 3.4.1 {
+  SELECT * FROM t4 
+  WHERE substr(a, 2) = 'abc' COLLATE NOCASE
+  ORDER BY substr(a, 2), b;
+} {
+  .ABC 1   .ABC 3   .abc 2   .abc 4
+}
+
+do_execsql_test 3.4.2 {
+  CREATE INDEX i4 ON t4( substr(a, 2) COLLATE NOCASE, b );
+  SELECT * FROM t4 
+  WHERE substr(a, 2) = 'abc' COLLATE NOCASE
+  ORDER BY substr(a, 2), b;
+} {
+  .ABC 1   .ABC 3   .abc 2   .abc 4
+}
+
 finish_test

From 0f3f7664f01bb9fdaa45c4c71c53fbff7f0f09c4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 18 Aug 2017 14:34:28 +0000
Subject: [PATCH 024/270] Combine the OP_CreateTable and OP_CreateIndex opcodes
 of the bytecode engine into a single OP_CreateBtree opcode.  This simplifies
 the implementation and makes the bytecode programs clearer.

FossilOrigin-Name: eb1202b5e43f1f029ad2bdf845509e7f31361e4dc189943e5e2bd4462e2ef3f3
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/build.c     | 17 ++++++++---------
 src/sqliteInt.h |  2 +-
 src/vdbe.c      | 44 +++++++++++---------------------------------
 src/vdbeaux.c   |  5 +++--
 6 files changed, 34 insertions(+), 56 deletions(-)

diff --git a/manifest b/manifest
index dad7e502b8..4fa7a63002 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Only\suse\sindexes\son\sexpressions\sto\soptimize\sORDER\sBY\sand\sGROUP\sBY\sif\sthe\ncollation\ssequence\smatches.\sPossible\sfix\sfor\s[e20dd54a].
-D 2017-08-18T08:29:37.727
+C Combine\sthe\sOP_CreateTable\sand\sOP_CreateIndex\sopcodes\sof\sthe\sbytecode\sengine\ninto\sa\ssingle\sOP_CreateBtree\sopcode.\s\sThis\ssimplifies\sthe\simplementation\sand\nmakes\sthe\sbytecode\sprograms\sclearer.
+D 2017-08-18T14:34:28.967
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -402,7 +402,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c e1a98bef27d6189df5c5b8b3c9b5e53d6b74bf9a936af97757c21e9ea6a5c6b2
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 5b81049a4cea3f547ddb4efc6f56345894524248816dc1ca1511b99be3f7d3ad
+F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -462,7 +462,7 @@ F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c17
 F src/sqlite.h.in f0bd1abf5e27bd22b3bcaae2a861c1efc4ab7e752bf7eb102355135617eb8199
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 6dddca4e215f4088aeaf60aebaa6d913397d61422733e160f25ab9dc53605a36
+F src/sqliteInt.h bc0792db4ff887e4884d386188527e1cd7c16d496c8f3ba23333e68cee3e4f78
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -524,11 +524,11 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 773843506ce694714bc96fe67c30c37015f90ef515d0e70f1f8d5c9c24088152
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 711a1b2c0a6046483ebf4ac43c4c0ea5bef3e50534a4ab9e1d7e3542635cd009
+F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
-F src/vdbeaux.c 1f15018ef7abe22669967f70b02bfe6709be403f126f713dabb091b9d631859a
+F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c b7fac20534c79b7554dab2e8a180c585a8bc1b9c85149d1b2d9746cf314d06ed
 F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e3
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a42a438cbbd721765ca55e71c464552dbaa494050cf472593599b8c7f0249516
-R 01b1b734d02df776d0a6e39431469431
-U dan
-Z 25c783941cfeb73ae08636afb42efa7c
+P 37e1900880b70be6802eaf43b0e568fda709a1dd6083d8be11e5a7a7d1fda41a
+R 590c7fe86e3ddb105e4572bf5c63dff2
+U drh
+Z 553a5b087246c1036f2341d36a71efe3
diff --git a/manifest.uuid b/manifest.uuid
index eae0feb8f2..e99f037cf1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-37e1900880b70be6802eaf43b0e568fda709a1dd6083d8be11e5a7a7d1fda41a
\ No newline at end of file
+eb1202b5e43f1f029ad2bdf845509e7f31361e4dc189943e5e2bd4462e2ef3f3
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 55407e77d9..05126f6260 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1010,7 +1010,8 @@ void sqlite3StartTable(
     }else
 #endif
     {
-      pParse->addrCrTab = sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2);
+      pParse->addrCrTab =
+         sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY);
     }
     sqlite3OpenMasterTable(pParse, iDb);
     sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1);
@@ -1670,9 +1671,8 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){
 ** Changes include:
 **
 **     (1)  Set all columns of the PRIMARY KEY schema object to be NOT NULL.
-**     (2)  Convert the OP_CreateTable into an OP_CreateIndex.  There is
-**          no rowid btree for a WITHOUT ROWID.  Instead, the canonical
-**          data storage is a covering index btree.
+**     (2)  Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY 
+**          into BTREE_BLOBKEY.
 **     (3)  Bypass the creation of the sqlite_master table entry
 **          for the PRIMARY KEY as the primary key index is now
 **          identified by the sqlite_master table entry of the table itself.
@@ -1709,13 +1709,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   ** virtual tables */
   if( IN_DECLARE_VTAB ) return;
 
-  /* Convert the OP_CreateTable opcode that would normally create the
-  ** root-page for the table into an OP_CreateIndex opcode.  The index
-  ** created will become the PRIMARY KEY index.
+  /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY
+  ** into BTREE_BLOBKEY.
   */
   if( pParse->addrCrTab ){
     assert( v );
-    sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex);
+    sqlite3VdbeChangeP3(v, pParse->addrCrTab, BTREE_BLOBKEY);
   }
 
   /* Locate the PRIMARY KEY index.  Or, if this table was originally
@@ -3366,7 +3365,7 @@ void sqlite3CreateIndex(
     ** that case the convertToWithoutRowidTable() routine will replace
     ** the Noop with a Goto to jump over the VDBE code generated below. */
     pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop);
-    sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem);
+    sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY);
 
     /* Gather the complete text of the CREATE INDEX statement into
     ** the zStmt variable
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8aa9832e8a..3689689403 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3001,7 +3001,7 @@ struct Parse {
   AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
   Parse *pToplevel;    /* Parse structure for main program (or NULL) */
   Table *pTriggerTab;  /* Table triggers are being coded for */
-  int addrCrTab;       /* Address of OP_CreateTable opcode on CREATE TABLE */
+  int addrCrTab;       /* Address of OP_CreateBtree opcode on CREATE TABLE */
   u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
   u32 oldmask;         /* Mask of old.* columns referenced */
   u32 newmask;         /* Mask of new.* columns referenced */
diff --git a/src/vdbe.c b/src/vdbe.c
index 3f5d967d2d..70e745afe0 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3409,7 +3409,7 @@ case OP_OpenWrite:
     assert( (pIn2->flags & MEM_Int)!=0 );
     sqlite3VdbeMemIntegerify(pIn2);
     p2 = (int)pIn2->u.i;
-    /* The p2 value always comes from a prior OP_CreateTable opcode and
+    /* The p2 value always comes from a prior OP_CreateBtree opcode and
     ** that opcode will always set the p2 value to 2 or more or else fail.
     ** If there were a failure, the prepared statement would have halted
     ** before reaching this instruction. */
@@ -5483,50 +5483,28 @@ case OP_ResetSorter: {
   break;
 }
 
-/* Opcode: CreateTable P1 P2 * * *
-** Synopsis: r[P2]=root iDb=P1
+/* Opcode: CreateBtree P1 P2 P3 * *
+** Synopsis: r[P2]=root iDb=P1 flags=P3
 **
-** Allocate a new table in the main database file if P1==0 or in the
-** auxiliary database file if P1==1 or in an attached database if
-** P1>1.  Write the root page number of the new table into
-** register P2
-**
-** The difference between a table and an index is this:  A table must
-** have a 4-byte integer key and can have arbitrary data.  An index
-** has an arbitrary key but no data.
-**
-** See also: CreateIndex
+** Allocate a new b-tree in the main database file if P1==0 or in the
+** TEMP database file if P1==1 or in an attached database if
+** P1>1.  The P3 argument must be 1 (BTREE_INTKEY) for a rowid table
+** it must be 2 (BTREE_BLOBKEY) for a index or WITHOUT ROWID table.
+** The root page number of the new b-tree is stored in register P2.
 */
-/* Opcode: CreateIndex P1 P2 * * *
-** Synopsis: r[P2]=root iDb=P1
-**
-** Allocate a new index in the main database file if P1==0 or in the
-** auxiliary database file if P1==1 or in an attached database if
-** P1>1.  Write the root page number of the new table into
-** register P2.
-**
-** See documentation on OP_CreateTable for additional information.
-*/
-case OP_CreateIndex:            /* out2 */
-case OP_CreateTable: {          /* out2 */
+case OP_CreateBtree: {          /* out2 */
   int pgno;
-  int flags;
   Db *pDb;
 
   pOut = out2Prerelease(p, pOp);
   pgno = 0;
+  assert( pOp->p3==BTREE_INTKEY || pOp->p3==BTREE_BLOBKEY );
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
   assert( DbMaskTest(p->btreeMask, pOp->p1) );
   assert( p->readOnly==0 );
   pDb = &db->aDb[pOp->p1];
   assert( pDb->pBt!=0 );
-  if( pOp->opcode==OP_CreateTable ){
-    /* flags = BTREE_INTKEY; */
-    flags = BTREE_INTKEY;
-  }else{
-    flags = BTREE_BLOBKEY;
-  }
-  rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags);
+  rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, pOp->p3);
   if( rc ) goto abort_due_to_error;
   pOut->u.i = pgno;
   break;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 7f6632bfa4..6de0efcb9e 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -490,7 +490,8 @@ static Op *opIterNext(VdbeOpIter *p){
 **   *  OP_VUpdate
 **   *  OP_VRename
 **   *  OP_FkCounter with P2==0 (immediate foreign key constraint)
-**   *  OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...)
+**   *  OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine 
+**      (for CREATE TABLE AS SELECT ...)
 **
 ** Then check that the value of Parse.mayAbort is true if an
 ** ABORT may be thrown, or false otherwise. Return true if it does
@@ -518,7 +519,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
       hasAbort = 1;
       break;
     }
-    if( opcode==OP_CreateTable ) hasCreateTable = 1;
+    if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1;
     if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1;
 #ifndef SQLITE_OMIT_FOREIGN_KEY
     if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){

From c68886bb9e02ff943ffe6481cb0e9097fd57111e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 18 Aug 2017 16:09:52 +0000
Subject: [PATCH 025/270] For the unix VFS, avoid an unnecessary stat() system
 call prior to opening any file in the common case where there are no unused
 file descriptors.

FossilOrigin-Name: 3075cfa07489eaf13cb9a2760e2391e79dd73181fe1730fae7bdcd6ad66d2a1f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c | 41 ++++++++++++++++++++++++-----------------
 3 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index 4fa7a63002..78a97342e5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Combine\sthe\sOP_CreateTable\sand\sOP_CreateIndex\sopcodes\sof\sthe\sbytecode\sengine\ninto\sa\ssingle\sOP_CreateBtree\sopcode.\s\sThis\ssimplifies\sthe\simplementation\sand\nmakes\sthe\sbytecode\sprograms\sclearer.
-D 2017-08-18T14:34:28.967
+C For\sthe\sunix\sVFS,\savoid\san\sunnecessary\sstat()\ssystem\scall\sprior\sto\sopening\nany\sfile\sin\sthe\scommon\scase\swhere\sthere\sare\sno\sunused\sfile\sdescriptors.
+D 2017-08-18T16:09:52.797
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -440,7 +440,7 @@ F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c a361273749229755f92c8f0e3e4855054ad39bbc5c65773e8db5d0b79afa632c
+F src/os_unix.c 0a7730f6cb797ba1fd12825e4ea751e1325041410c063c258e30089ca71f9a88
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 6f2ae58c0d4ddf510d324cb2ec38f471b5cff8f3e061afd32717ad790685cc7f
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 37e1900880b70be6802eaf43b0e568fda709a1dd6083d8be11e5a7a7d1fda41a
-R 590c7fe86e3ddb105e4572bf5c63dff2
+P eb1202b5e43f1f029ad2bdf845509e7f31361e4dc189943e5e2bd4462e2ef3f3
+R 75fd277ce5c40358998ed365a587d991
 U drh
-Z 553a5b087246c1036f2341d36a71efe3
+Z 8da9bd8ec0af0fa3e80213d960cc13a7
diff --git a/manifest.uuid b/manifest.uuid
index e99f037cf1..97cbaa15bb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eb1202b5e43f1f029ad2bdf845509e7f31361e4dc189943e5e2bd4462e2ef3f3
\ No newline at end of file
+3075cfa07489eaf13cb9a2760e2391e79dd73181fe1730fae7bdcd6ad66d2a1f
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index d1ebd81f68..bd646d6e70 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -210,7 +210,7 @@ struct unixFile {
   unsigned short int ctrlFlags;       /* Behavioral bits.  UNIXFILE_* flags */
   int lastErrno;                      /* The unix errno from last I/O error */
   void *lockingContext;               /* Locking style specific state */
-  UnixUnusedFd *pUnused;              /* Pre-allocated UnixUnusedFd */
+  UnixUnusedFd *pPreallocatedUnused;  /* Pre-allocated UnixUnusedFd */
   const char *zPath;                  /* Name of the file */
   unixShm *pShm;                      /* Shared memory segment information */
   int szChunk;                        /* Configured by FCNTL_CHUNK_SIZE */
@@ -1120,7 +1120,8 @@ struct unixInodeInfo {
 /*
 ** A lists of all unixInodeInfo objects.
 */
-static unixInodeInfo *inodeList = 0;
+static unixInodeInfo *inodeList = 0;  /* All unixInodeInfo objects */
+static unsigned int nUnusedFd = 0;    /* Total unused file descriptors */
 
 /*
 **
@@ -1230,6 +1231,7 @@ static void closePendingFds(unixFile *pFile){
     pNext = p->pNext;
     robust_close(pFile, p->fd, __LINE__);
     sqlite3_free(p);
+    nUnusedFd--;
   }
   pInode->pUnused = 0;
 }
@@ -1262,6 +1264,7 @@ static void releaseInodeInfo(unixFile *pFile){
       sqlite3_free(pInode);
     }
   }
+  assert( inodeList!=0 || nUnusedFd==0 );
 }
 
 /*
@@ -1331,6 +1334,7 @@ static int findInodeInfo(
 #else
   fileId.ino = (u64)statbuf.st_ino;
 #endif
+  assert( inodeList!=0 || nUnusedFd==0 );
   pInode = inodeList;
   while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){
     pInode = pInode->pNext;
@@ -1750,11 +1754,12 @@ end_lock:
 */
 static void setPendingFd(unixFile *pFile){
   unixInodeInfo *pInode = pFile->pInode;
-  UnixUnusedFd *p = pFile->pUnused;
+  UnixUnusedFd *p = pFile->pPreallocatedUnused;
   p->pNext = pInode->pUnused;
   pInode->pUnused = p;
   pFile->h = -1;
-  pFile->pUnused = 0;
+  pFile->pPreallocatedUnused = 0;
+  nUnusedFd++;
 }
 
 /*
@@ -1979,7 +1984,7 @@ static int closeUnixFile(sqlite3_file *id){
 #endif
   OSTRACE(("CLOSE   %-3d\n", pFile->h));
   OpenCounter(-1);
-  sqlite3_free(pFile->pUnused);
+  sqlite3_free(pFile->pPreallocatedUnused);
   memset(pFile, 0, sizeof(unixFile));
   return SQLITE_OK;
 }
@@ -3200,7 +3205,7 @@ static int unixRead(
   /* If this is a database file (not a journal, master-journal or temp
   ** file), the bytes in the locking range should never be read or written. */
 #if 0
-  assert( pFile->pUnused==0
+  assert( pFile->pPreallocatedUnused==0
        || offset>=PENDING_BYTE+512
        || offset+amt<=PENDING_BYTE 
   );
@@ -3313,7 +3318,7 @@ static int unixWrite(
   /* If this is a database file (not a journal, master-journal or temp
   ** file), the bytes in the locking range should never be read or written. */
 #if 0
-  assert( pFile->pUnused==0
+  assert( pFile->pPreallocatedUnused==0
        || offset>=PENDING_BYTE+512
        || offset+amt<=PENDING_BYTE 
   );
@@ -5564,6 +5569,8 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){
 #if !OS_VXWORKS
   struct stat sStat;                   /* Results of stat() call */
 
+  unixEnterMutex();
+
   /* A stat() call may fail for various reasons. If this happens, it is
   ** almost certain that an open() call on the same path will also fail.
   ** For this reason, if an error occurs in the stat() call here, it is
@@ -5572,10 +5579,9 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){
   **
   ** Even if a subsequent open() call does succeed, the consequences of
   ** not searching for a reusable file descriptor are not dire.  */
-  if( 0==osStat(zPath, &sStat) ){
+  if( nUnusedFd>0 && 0==osStat(zPath, &sStat) ){
     unixInodeInfo *pInode;
 
-    unixEnterMutex();
     pInode = inodeList;
     while( pInode && (pInode->fileId.dev!=sStat.st_dev
                      || pInode->fileId.ino!=(u64)sStat.st_ino) ){
@@ -5586,11 +5592,12 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){
       for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext));
       pUnused = *pp;
       if( pUnused ){
+        nUnusedFd--;
         *pp = pUnused->pNext;
       }
     }
-    unixLeaveMutex();
   }
+  unixLeaveMutex();
 #endif    /* if !OS_VXWORKS */
   return pUnused;
 }
@@ -5811,7 +5818,7 @@ static int unixOpen(
         return SQLITE_NOMEM_BKPT;
       }
     }
-    p->pUnused = pUnused;
+    p->pPreallocatedUnused = pUnused;
 
     /* Database filenames are double-zero terminated if they are not
     ** URIs with parameters.  Hence, they can always be passed into
@@ -5848,7 +5855,7 @@ static int unixOpen(
     gid_t gid;                    /* Groupid for the file */
     rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid);
     if( rc!=SQLITE_OK ){
-      assert( !p->pUnused );
+      assert( !p->pPreallocatedUnused );
       assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL );
       return rc;
     }
@@ -5882,9 +5889,9 @@ static int unixOpen(
     *pOutFlags = flags;
   }
 
-  if( p->pUnused ){
-    p->pUnused->fd = fd;
-    p->pUnused->flags = flags;
+  if( p->pPreallocatedUnused ){
+    p->pPreallocatedUnused->fd = fd;
+    p->pPreallocatedUnused->flags = flags;
   }
 
   if( isDelete ){
@@ -5965,7 +5972,7 @@ static int unixOpen(
 
 open_finished:
   if( rc!=SQLITE_OK ){
-    sqlite3_free(p->pUnused);
+    sqlite3_free(p->pPreallocatedUnused);
   }
   return rc;
 }
@@ -6706,7 +6713,7 @@ static int proxyCreateUnixFile(
   dummyVfs.zName = "dummy";
   pUnused->fd = fd;
   pUnused->flags = openFlags;
-  pNew->pUnused = pUnused;
+  pNew->pPreallocatedUnused = pUnused;
   
   rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0);
   if( rc==SQLITE_OK ){

From 56520ab8487200bb697911baa864d7a996ab17f1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 18 Aug 2017 21:14:50 +0000
Subject: [PATCH 026/270] Size and performance optimization the readDbPage()
 routine in the pager.

FossilOrigin-Name: ca9e1875c3a893321d70a131fc4ffc76d169ad05e0b48b7006f53b6b467db4be
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   | 29 +++++++++++------------------
 3 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index 78a97342e5..1e5d32d66b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C For\sthe\sunix\sVFS,\savoid\san\sunnecessary\sstat()\ssystem\scall\sprior\sto\sopening\nany\sfile\sin\sthe\scommon\scase\swhere\sthere\sare\sno\sunused\sfile\sdescriptors.
-D 2017-08-18T16:09:52.797
+C Size\sand\sperformance\soptimization\sthe\sreadDbPage()\sroutine\sin\sthe\spager.
+D 2017-08-18T21:14:50.622
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -443,7 +443,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 0a7730f6cb797ba1fd12825e4ea751e1325041410c063c258e30089ca71f9a88
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 6f2ae58c0d4ddf510d324cb2ec38f471b5cff8f3e061afd32717ad790685cc7f
+F src/pager.c fb9a8f40417d6dfbd6b8be91237f2f64d51cc867ab28687420acbc5cab786a3c
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P eb1202b5e43f1f029ad2bdf845509e7f31361e4dc189943e5e2bd4462e2ef3f3
-R 75fd277ce5c40358998ed365a587d991
+P 3075cfa07489eaf13cb9a2760e2391e79dd73181fe1730fae7bdcd6ad66d2a1f
+R 73e91e0689b3737aa34e053426722c6a
 U drh
-Z 8da9bd8ec0af0fa3e80213d960cc13a7
+Z 58f532a67b173dfa15cad7f44c1bc81c
diff --git a/manifest.uuid b/manifest.uuid
index 97cbaa15bb..59813005d8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3075cfa07489eaf13cb9a2760e2391e79dd73181fe1730fae7bdcd6ad66d2a1f
\ No newline at end of file
+ca9e1875c3a893321d70a131fc4ffc76d169ad05e0b48b7006f53b6b467db4be
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 1677dfcda7..9a430476a1 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -2982,7 +2982,8 @@ end_playback:
 
 
 /*
-** Read the content for page pPg out of the database file and into 
+** Read the content for page pPg out of the database file (or out of
+** the WAL if that is where the most recent copy if found) into 
 ** pPg->pData. A shared lock or greater must be held on the database
 ** file before this function is called.
 **
@@ -2992,22 +2993,23 @@ end_playback:
 ** If an IO error occurs, then the IO error is returned to the caller.
 ** Otherwise, SQLITE_OK is returned.
 */
-static int readDbPage(PgHdr *pPg, u32 iFrame){
+static int readDbPage(PgHdr *pPg){
   Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
   Pgno pgno = pPg->pgno;       /* Page number to read */
   int rc = SQLITE_OK;          /* Return code */
   int pgsz = pPager->pageSize; /* Number of bytes to read */
+  u32 iFrame = 0;              /* Frame of WAL containing pgno */
 
   assert( pPager->eState>=PAGER_READER && !MEMDB );
   assert( isOpen(pPager->fd) );
 
-#ifndef SQLITE_OMIT_WAL
+  if( pagerUseWal(pPager) ){
+    rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame);
+    if( rc ) return rc;
+  }
   if( iFrame ){
-    /* Try to pull the page from the write-ahead log. */
     rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData);
-  }else
-#endif
-  {
+  }else{
     i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
     rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
     if( rc==SQLITE_IOERR_SHORT_READ ){
@@ -3092,11 +3094,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){
     if( sqlite3PcachePageRefcount(pPg)==1 ){
       sqlite3PcacheDrop(pPg);
     }else{
-      u32 iFrame = 0;
-      rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame);
-      if( rc==SQLITE_OK ){
-        rc = readDbPage(pPg, iFrame);
-      }
+      rc = readDbPage(pPg);
       if( rc==SQLITE_OK ){
         pPager->xReiniter(pPg);
       }
@@ -5489,14 +5487,9 @@ static int getPageNormal(
       memset(pPg->pData, 0, pPager->pageSize);
       IOTRACE(("ZERO %p %d\n", pPager, pgno));
     }else{
-      u32 iFrame = 0;                 /* Frame to read from WAL file */
-      if( pagerUseWal(pPager) ){
-        rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame);
-        if( rc!=SQLITE_OK ) goto pager_acquire_err;
-      }
       assert( pPg->pPager==pPager );
       pPager->aStat[PAGER_STAT_MISS]++;
-      rc = readDbPage(pPg, iFrame);
+      rc = readDbPage(pPg);
       if( rc!=SQLITE_OK ){
         goto pager_acquire_err;
       }

From 251866d07c0b76e4fd0d762febe3f37a559f961a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 18 Aug 2017 22:30:20 +0000
Subject: [PATCH 027/270] Another size and performance optimization to
 readDbPage().  This time we eliminate some unnecessary local variables.

FossilOrigin-Name: 745bc8decd18d4dc00589474fd3928a3a9f4156d09e05e6f5b8623de6491795a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   | 18 ++++++++----------
 3 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 1e5d32d66b..558153cf2f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\sand\sperformance\soptimization\sthe\sreadDbPage()\sroutine\sin\sthe\spager.
-D 2017-08-18T21:14:50.622
+C Another\ssize\sand\sperformance\soptimization\sto\sreadDbPage().\s\sThis\stime\swe\neliminate\ssome\sunnecessary\slocal\svariables.
+D 2017-08-18T22:30:20.164
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -443,7 +443,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 0a7730f6cb797ba1fd12825e4ea751e1325041410c063c258e30089ca71f9a88
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c fb9a8f40417d6dfbd6b8be91237f2f64d51cc867ab28687420acbc5cab786a3c
+F src/pager.c c1dc0609f04a0659519bb2b8ca1440a64b0ad82b6c2afd675f1a50f6c918321a
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3075cfa07489eaf13cb9a2760e2391e79dd73181fe1730fae7bdcd6ad66d2a1f
-R 73e91e0689b3737aa34e053426722c6a
+P ca9e1875c3a893321d70a131fc4ffc76d169ad05e0b48b7006f53b6b467db4be
+R 2ae3b6889f1fe566e9272b8874e17865
 U drh
-Z 58f532a67b173dfa15cad7f44c1bc81c
+Z 970c0b65cb119d003a12faf3a1e4e18e
diff --git a/manifest.uuid b/manifest.uuid
index 59813005d8..6c04ddf6a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ca9e1875c3a893321d70a131fc4ffc76d169ad05e0b48b7006f53b6b467db4be
\ No newline at end of file
+745bc8decd18d4dc00589474fd3928a3a9f4156d09e05e6f5b8623de6491795a
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 9a430476a1..5986afdae1 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -2995,29 +2995,27 @@ end_playback:
 */
 static int readDbPage(PgHdr *pPg){
   Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
-  Pgno pgno = pPg->pgno;       /* Page number to read */
   int rc = SQLITE_OK;          /* Return code */
-  int pgsz = pPager->pageSize; /* Number of bytes to read */
   u32 iFrame = 0;              /* Frame of WAL containing pgno */
 
   assert( pPager->eState>=PAGER_READER && !MEMDB );
   assert( isOpen(pPager->fd) );
 
   if( pagerUseWal(pPager) ){
-    rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame);
+    rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame);
     if( rc ) return rc;
   }
   if( iFrame ){
-    rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData);
+    rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData);
   }else{
-    i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
-    rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
+    i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize;
+    rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset);
     if( rc==SQLITE_IOERR_SHORT_READ ){
       rc = SQLITE_OK;
     }
   }
 
-  if( pgno==1 ){
+  if( pPg->pgno==1 ){
     if( rc ){
       /* If the read is unsuccessful, set the dbFileVers[] to something
       ** that will never be a valid file version.  dbFileVers[] is a copy
@@ -3037,13 +3035,13 @@ static int readDbPage(PgHdr *pPg){
       memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers));
     }
   }
-  CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT);
+  CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM_BKPT);
 
   PAGER_INCR(sqlite3_pager_readdb_count);
   PAGER_INCR(pPager->nRead);
-  IOTRACE(("PGIN %p %d\n", pPager, pgno));
+  IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno));
   PAGERTRACE(("FETCH %d page %d hash(%08x)\n",
-               PAGERID(pPager), pgno, pager_pagehash(pPg)));
+               PAGERID(pPager), pPg->pgno, pager_pagehash(pPg)));
 
   return rc;
 }

From c84ddf14c5c65007b739d4368cdde7fb24eaed78 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 19 Aug 2017 20:38:18 +0000
Subject: [PATCH 028/270] Space and size optimization to the printf
 implementation.

FossilOrigin-Name: d01d2cffefd1cdb52b386e4983599534c0fbbe6aebda186db53200e4b2283f0a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/printf.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 558153cf2f..19ef566777 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Another\ssize\sand\sperformance\soptimization\sto\sreadDbPage().\s\sThis\stime\swe\neliminate\ssome\sunnecessary\slocal\svariables.
-D 2017-08-18T22:30:20.164
+C Space\sand\ssize\soptimization\sto\sthe\sprintf\simplementation.
+D 2017-08-19T20:38:18.374
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -452,7 +452,7 @@ F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
 F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9e880c0efb5d7f9101bb34c0a87daf6e1e5284c34024fdb811e67bb02fdd299b
-F src/printf.c 439c145d71ff9e0328ec26b7db23a45f995f742d3207dd546e034cdc1667ac9c
+F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ca9e1875c3a893321d70a131fc4ffc76d169ad05e0b48b7006f53b6b467db4be
-R 2ae3b6889f1fe566e9272b8874e17865
+P 745bc8decd18d4dc00589474fd3928a3a9f4156d09e05e6f5b8623de6491795a
+R f17a35ca8aefa46f1d59e82e2ca24c65
 U drh
-Z 970c0b65cb119d003a12faf3a1e4e18e
+Z 9e343ccf54a229e4a80d7dbe62c9be25
diff --git a/manifest.uuid b/manifest.uuid
index 6c04ddf6a7..467969b46f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-745bc8decd18d4dc00589474fd3928a3a9f4156d09e05e6f5b8623de6491795a
\ No newline at end of file
+d01d2cffefd1cdb52b386e4983599534c0fbbe6aebda186db53200e4b2283f0a
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index 49b13cc4f5..9427844e09 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -656,7 +656,7 @@ void sqlite3VXPrintf(
         if( precision>=0 ){
           for(length=0; length<precision && bufpt[length]; length++){}
         }else{
-          length = sqlite3Strlen30(bufpt);
+          length = 0x7fffffff & (int)strlen(bufpt);
         }
         break;
       case etSQLESCAPE:           /* Escape ' characters */

From b32c18bf42cf910dae1bed3059d03a037b85db51 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 21 Aug 2017 02:05:22 +0000
Subject: [PATCH 029/270] Minor optimization to sqlite3VdbeMemSetStr().

FossilOrigin-Name: 6538ef7b6b56c7a6629a0bb7418910c64c8b2e73af2296a116c073ecf2e0d429
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 19ef566777..cac92f9d02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Space\sand\ssize\soptimization\sto\sthe\sprintf\simplementation.
-D 2017-08-19T20:38:18.374
+C Minor\soptimization\sto\ssqlite3VdbeMemSetStr().
+D 2017-08-21T02:05:22.118
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -530,7 +530,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
 F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
-F src/vdbemem.c b7fac20534c79b7554dab2e8a180c585a8bc1b9c85149d1b2d9746cf314d06ed
+F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
 F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e3
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 745bc8decd18d4dc00589474fd3928a3a9f4156d09e05e6f5b8623de6491795a
-R f17a35ca8aefa46f1d59e82e2ca24c65
+P d01d2cffefd1cdb52b386e4983599534c0fbbe6aebda186db53200e4b2283f0a
+R 4e15eb10124cc2a73cc86451db58097f
 U drh
-Z 9e343ccf54a229e4a80d7dbe62c9be25
+Z cde05a24b81c141b2ecb8ef40de25a03
diff --git a/manifest.uuid b/manifest.uuid
index 467969b46f..fc8c38515c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d01d2cffefd1cdb52b386e4983599534c0fbbe6aebda186db53200e4b2283f0a
\ No newline at end of file
+6538ef7b6b56c7a6629a0bb7418910c64c8b2e73af2296a116c073ecf2e0d429
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 346c1fb10b..345c41a72d 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -931,7 +931,7 @@ int sqlite3VdbeMemSetStr(
   if( nByte<0 ){
     assert( enc!=0 );
     if( enc==SQLITE_UTF8 ){
-      nByte = sqlite3Strlen30(z);
+      nByte = 0x7fffffff & (int)strlen(z);
       if( nByte>iLimit ) nByte = iLimit+1;
     }else{
       for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){}

From b40f06c62dc64e516510b8221950e1d6b687f7ee Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 21 Aug 2017 02:20:57 +0000
Subject: [PATCH 030/270] Remove an unnecessary conditional.

FossilOrigin-Name: 56d19f9fd7b01d4ed5c3e7309977b43fedffee168c9760d3e3b7e885790f781e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/expr.c    |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index cac92f9d02..9ae22d4f8e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\soptimization\sto\ssqlite3VdbeMemSetStr().
-D 2017-08-21T02:05:22.118
+C Remove\san\sunnecessary\sconditional.
+D 2017-08-21T02:20:57.510
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -409,7 +409,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c dc436431dc50a0256b9dcd3daaa06aac0df21834f91068525f2eb3c10b9a7a9a
+F src/expr.c 4ca86dc65f5ea478c665a5b4fe79d05f00432c9bd82237a896b45bd376bf1217
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c ed8888ae80b39f5a5d403954e4a05e0a38303523dff8143161439c142d31dec1
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d01d2cffefd1cdb52b386e4983599534c0fbbe6aebda186db53200e4b2283f0a
-R 4e15eb10124cc2a73cc86451db58097f
+P 6538ef7b6b56c7a6629a0bb7418910c64c8b2e73af2296a116c073ecf2e0d429
+R 44db9cb33f37af7df65ab136e59cd109
 U drh
-Z cde05a24b81c141b2ecb8ef40de25a03
+Z bcd0fe0605e077fa72b238dc51f819ad
diff --git a/manifest.uuid b/manifest.uuid
index fc8c38515c..40286a0698 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6538ef7b6b56c7a6629a0bb7418910c64c8b2e73af2296a116c073ecf2e0d429
\ No newline at end of file
+56d19f9fd7b01d4ed5c3e7309977b43fedffee168c9760d3e3b7e885790f781e
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index d090aab3b7..0e1a8781e1 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -775,7 +775,7 @@ Expr *sqlite3Expr(
 ){
   Token x;
   x.z = zToken;
-  x.n = zToken ? sqlite3Strlen30(zToken) : 0;
+  x.n = sqlite3Strlen30(zToken);
   return sqlite3ExprAlloc(db, op, &x, 0);
 }
 

From 2e2338101a28a647eaa37cf20945087fd96aaf10 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 15:21:54 +0000
Subject: [PATCH 031/270] Fix error tests in seldom-used compile-time branches
 of the unix backend.

FossilOrigin-Name: 885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 9ae22d4f8e..a66cc1c8aa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\sconditional.
-D 2017-08-21T02:20:57.510
+C Fix\serror\stests\sin\sseldom-used\scompile-time\sbranches\sof\sthe\sunix\sbackend.
+D 2017-08-22T15:21:54.760
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -440,7 +440,7 @@ F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 0a7730f6cb797ba1fd12825e4ea751e1325041410c063c258e30089ca71f9a88
+F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c c1dc0609f04a0659519bb2b8ca1440a64b0ad82b6c2afd675f1a50f6c918321a
@@ -1649,7 +1649,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6538ef7b6b56c7a6629a0bb7418910c64c8b2e73af2296a116c073ecf2e0d429
-R 44db9cb33f37af7df65ab136e59cd109
+P 56d19f9fd7b01d4ed5c3e7309977b43fedffee168c9760d3e3b7e885790f781e
+R fadb89000aeb50524d087e05d444ff10
 U drh
-Z bcd0fe0605e077fa72b238dc51f819ad
+Z b11624f2af9f39a4604e682002fd5d5e
diff --git a/manifest.uuid b/manifest.uuid
index 40286a0698..9cd165bc2c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-56d19f9fd7b01d4ed5c3e7309977b43fedffee168c9760d3e3b7e885790f781e
\ No newline at end of file
+885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index bd646d6e70..0d7e494147 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -2321,7 +2321,7 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){
   OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved));
 
 #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS
-  if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){
+  if( (rc & 0xff) == SQLITE_IOERR ){
     rc = SQLITE_OK;
     reserved=1;
   }
@@ -2388,7 +2388,7 @@ static int flockLock(sqlite3_file *id, int eFileLock) {
   OSTRACE(("LOCK    %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), 
            rc==SQLITE_OK ? "ok" : "failed"));
 #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS
-  if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){
+  if( (rc & 0xff) == SQLITE_IOERR ){
     rc = SQLITE_BUSY;
   }
 #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */
@@ -2925,7 +2925,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){
           /* Can't reestablish the shared lock.  Sqlite can't deal, this is
           ** a critical I/O error
           */
-          rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : 
+          rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : 
                SQLITE_IOERR_LOCK;
           goto afp_end_lock;
         } 

From f39e0ed4de8d76c458bc8c3cf758637b6720e30c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 19:19:00 +0000
Subject: [PATCH 032/270] Add the "mksourceid" program to the build process. 
 That program changes the SQLITE_SOURCE_ID if the source tree has been
 modified in any way.

FossilOrigin-Name: d4c05e04f7e1325a3260808ee17252876f678e78bf0cf6569a18a52ff674bd7a
---
 Makefile.in         |   7 +-
 Makefile.msc        |   8 +-
 main.mk             |   7 +-
 manifest            |  19 +-
 manifest.uuid       |   2 +-
 tool/mksourceid.c   | 852 ++++++++++++++++++++++++++++++++++++++++++++
 tool/mksqlite3h.tcl |  21 +-
 7 files changed, 885 insertions(+), 31 deletions(-)
 create mode 100644 tool/mksourceid.c

diff --git a/Makefile.in b/Makefile.in
index bce57b717b..50cd5e8e23 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -693,6 +693,11 @@ lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
 	cp $(TOP)/tool/lempar.c .
 
+# Rules to build the program that generates the source-id
+#
+mksourceid$(BEXE):	$(TOP)/tool/mksourceid.c
+	$(BCC) -o $@ $(TOP)/tool/mksourceid.c
+
 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
 #
@@ -958,7 +963,7 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
 	mv parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
 
-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
 
 keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
diff --git a/Makefile.msc b/Makefile.msc
index da94288c85..a1937fe5fa 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1670,6 +1670,12 @@ lemon.exe:	$(TOP)\tool\lemon.c lempar.c
 	$(BCC) $(NO_WARN) -Daccess=_access \
 		-Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
 
+# <<mark>>
+# Rules to build the source-id generator tool
+#
+mksourceid.exe:	$(TOP)\tool\mksourceid.c
+	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\mksourceid.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
+
 # Rules to build individual *.lo files from generated *.c files. This
 # applies to:
 #
@@ -1948,7 +1954,7 @@ parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
 	move parse.h parse.h.temp
 	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
 
-$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
+$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION
 	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)
 
 sqlite3ext.h:	.target_source
diff --git a/main.mk b/main.mk
index d3450a63cb..054f6518a5 100644
--- a/main.mk
+++ b/main.mk
@@ -606,6 +606,11 @@ lemon:	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o lemon $(TOP)/tool/lemon.c
 	cp $(TOP)/tool/lempar.c .
 
+# A tool to generate the source-id
+#
+mksourceid:	$(TOP)/tool/mksourceid.c
+	$(BCC) -o mksourceid $(TOP)/tool/mksourceid.c
+
 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
 #
@@ -645,7 +650,7 @@ parse.c:	$(TOP)/src/parse.y lemon $(TOP)/tool/addopcodes.tcl
 	mv parse.h parse.h.temp
 	tclsh $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
 
-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h
 	tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
 
 keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
diff --git a/manifest b/manifest
index a66cc1c8aa..6f9319811f 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\serror\stests\sin\sseldom-used\scompile-time\sbranches\sof\sthe\sunix\sbackend.
-D 2017-08-22T15:21:54.760
-F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
+C Add\sthe\s"mksourceid"\sprogram\sto\sthe\sbuild\sprocess.\s\sThat\sprogram\schanges\nthe\sSQLITE_SOURCE_ID\sif\sthe\ssource\stree\shas\sbeen\smodified\sin\sany\sway.
+D 2017-08-22T19:19:00.280
+F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
+F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -380,7 +380,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 5b7d72ab03dd70aa1401f934d31e85aefd6fc542eb58094d7a95d6921390b2d0
+F main.mk da75a0527a56da0b7f568a976b3cb69756613080f16e4d208b6c6a0495bfb132
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1588,10 +1588,11 @@ F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
 F tool/mkshellc.tcl 69c38ecd7b74b2b0799a35ce20e1e3998e504d8c99c100ca4b98ae9d8f6279bc
+F tool/mksourceid.c 1db4636bf4249ac5d2d56fb06b0b30929bf7446a88c7048bb731a72a1c4767ea
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
 F tool/mksqlite3c.tcl a4b36eaa002ed00a0ab2c93d999a14f1acae98ff09a85382e5abc05a91edb82b
-F tool/mksqlite3h.tcl 51bd5e7e840a920388a5966c9f2ccc618f434c57bd68c1bab4085b2553e1e237
+F tool/mksqlite3h.tcl e10b7878ca20161594fbc28913027956d7ee35713400534a115d8f139b496478
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
@@ -1649,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 56d19f9fd7b01d4ed5c3e7309977b43fedffee168c9760d3e3b7e885790f781e
-R fadb89000aeb50524d087e05d444ff10
+P 885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254
+R 48a68cf4cf699ac46128e2797c5425f9
 U drh
-Z b11624f2af9f39a4604e682002fd5d5e
+Z 40e98cab2922888e2c30394283edf487
diff --git a/manifest.uuid b/manifest.uuid
index 9cd165bc2c..616786c10c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254
\ No newline at end of file
+d4c05e04f7e1325a3260808ee17252876f678e78bf0cf6569a18a52ff674bd7a
\ No newline at end of file
diff --git a/tool/mksourceid.c b/tool/mksourceid.c
new file mode 100644
index 0000000000..722728be4c
--- /dev/null
+++ b/tool/mksourceid.c
@@ -0,0 +1,852 @@
+/*
+** Run this program with a single argument which is the name of the
+** Fossil "manifest" file for a project, and this program will emit on
+** standard output the "source id" for for the program.
+**
+** (1)  The "source id" is the date of check-in together with the 
+**      SHA3 hash of the manifest file.
+**
+** (2)  All individual file hashes in the manifest are verified.  If any
+**      source file has changed, the SHA3 hash ends with "-modified".
+**
+*/
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <ctype.h>
+
+/* Portable 64-bit unsigned integers */
+#if defined(_MSC_VER) || defined(__BORLANDC__)
+  typedef unsigned __int64 u64;
+#else
+  typedef unsigned long long int u64;
+#endif
+
+
+/*
+** Macros to determine whether the machine is big or little endian,
+** and whether or not that determination is run-time or compile-time.
+**
+** For best performance, an attempt is made to guess at the byte-order
+** using C-preprocessor macros.  If that is unsuccessful, or if
+** -DBYTEORDER=0 is set, then byte-order is determined
+** at run-time.
+*/
+#ifndef BYTEORDER
+# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+     defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
+     defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
+     defined(__arm__)
+#   define BYTEORDER    1234
+# elif defined(sparc)    || defined(__ppc__)
+#   define BYTEORDER    4321
+# else
+#   define BYTEORDER 0
+# endif
+#endif
+
+
+
+/*
+** State structure for a SHA3 hash in progress
+*/
+typedef struct SHA3Context SHA3Context;
+struct SHA3Context {
+  union {
+    u64 s[25];                /* Keccak state. 5x5 lines of 64 bits each */
+    unsigned char x[1600];    /* ... or 1600 bytes */
+  } u;
+  unsigned nRate;        /* Bytes of input accepted per Keccak iteration */
+  unsigned nLoaded;      /* Input bytes loaded into u.x[] so far this cycle */
+  unsigned ixMask;       /* Insert next input into u.x[nLoaded^ixMask]. */
+};
+
+/*
+** A single step of the Keccak mixing function for a 1600-bit state
+*/
+static void KeccakF1600Step(SHA3Context *p){
+  int i;
+  u64 B0, B1, B2, B3, B4;
+  u64 C0, C1, C2, C3, C4;
+  u64 D0, D1, D2, D3, D4;
+  static const u64 RC[] = {
+    0x0000000000000001ULL,  0x0000000000008082ULL,
+    0x800000000000808aULL,  0x8000000080008000ULL,
+    0x000000000000808bULL,  0x0000000080000001ULL,
+    0x8000000080008081ULL,  0x8000000000008009ULL,
+    0x000000000000008aULL,  0x0000000000000088ULL,
+    0x0000000080008009ULL,  0x000000008000000aULL,
+    0x000000008000808bULL,  0x800000000000008bULL,
+    0x8000000000008089ULL,  0x8000000000008003ULL,
+    0x8000000000008002ULL,  0x8000000000000080ULL,
+    0x000000000000800aULL,  0x800000008000000aULL,
+    0x8000000080008081ULL,  0x8000000000008080ULL,
+    0x0000000080000001ULL,  0x8000000080008008ULL
+  };
+# define A00 (p->u.s[0])
+# define A01 (p->u.s[1])
+# define A02 (p->u.s[2])
+# define A03 (p->u.s[3])
+# define A04 (p->u.s[4])
+# define A10 (p->u.s[5])
+# define A11 (p->u.s[6])
+# define A12 (p->u.s[7])
+# define A13 (p->u.s[8])
+# define A14 (p->u.s[9])
+# define A20 (p->u.s[10])
+# define A21 (p->u.s[11])
+# define A22 (p->u.s[12])
+# define A23 (p->u.s[13])
+# define A24 (p->u.s[14])
+# define A30 (p->u.s[15])
+# define A31 (p->u.s[16])
+# define A32 (p->u.s[17])
+# define A33 (p->u.s[18])
+# define A34 (p->u.s[19])
+# define A40 (p->u.s[20])
+# define A41 (p->u.s[21])
+# define A42 (p->u.s[22])
+# define A43 (p->u.s[23])
+# define A44 (p->u.s[24])
+# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
+
+  for(i=0; i<24; i+=4){
+    C0 = A00^A10^A20^A30^A40;
+    C1 = A01^A11^A21^A31^A41;
+    C2 = A02^A12^A22^A32^A42;
+    C3 = A03^A13^A23^A33^A43;
+    C4 = A04^A14^A24^A34^A44;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A11^D1), 44);
+    B2 = ROL64((A22^D2), 43);
+    B3 = ROL64((A33^D3), 21);
+    B4 = ROL64((A44^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i];
+    A11 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A20^D0), 3);
+    B3 = ROL64((A31^D1), 45);
+    B4 = ROL64((A42^D2), 61);
+    B0 = ROL64((A03^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A20 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A40^D0), 18);
+    B0 = ROL64((A01^D1), 1);
+    B1 = ROL64((A12^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A34^D4), 8);
+    A40 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A10^D0), 36);
+    B2 = ROL64((A21^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A43^D3), 56);
+    B0 = ROL64((A04^D4), 27);
+    A10 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A30^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A02^D2), 62);
+    B1 = ROL64((A13^D3), 55);
+    B2 = ROL64((A24^D4), 39);
+    A30 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A20^A40^A10^A30;
+    C1 = A11^A31^A01^A21^A41;
+    C2 = A22^A42^A12^A32^A02;
+    C3 = A33^A03^A23^A43^A13;
+    C4 = A44^A14^A34^A04^A24;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A31^D1), 44);
+    B2 = ROL64((A12^D2), 43);
+    B3 = ROL64((A43^D3), 21);
+    B4 = ROL64((A24^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+1];
+    A31 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A40^D0), 3);
+    B3 = ROL64((A21^D1), 45);
+    B4 = ROL64((A02^D2), 61);
+    B0 = ROL64((A33^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A40 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A30^D0), 18);
+    B0 = ROL64((A11^D1), 1);
+    B1 = ROL64((A42^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A04^D4), 8);
+    A30 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A20^D0), 36);
+    B2 = ROL64((A01^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A13^D3), 56);
+    B0 = ROL64((A44^D4), 27);
+    A20 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A10^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A22^D2), 62);
+    B1 = ROL64((A03^D3), 55);
+    B2 = ROL64((A34^D4), 39);
+    A10 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A40^A30^A20^A10;
+    C1 = A31^A21^A11^A01^A41;
+    C2 = A12^A02^A42^A32^A22;
+    C3 = A43^A33^A23^A13^A03;
+    C4 = A24^A14^A04^A44^A34;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A21^D1), 44);
+    B2 = ROL64((A42^D2), 43);
+    B3 = ROL64((A13^D3), 21);
+    B4 = ROL64((A34^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+2];
+    A21 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A30^D0), 3);
+    B3 = ROL64((A01^D1), 45);
+    B4 = ROL64((A22^D2), 61);
+    B0 = ROL64((A43^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A30 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A10^D0), 18);
+    B0 = ROL64((A31^D1), 1);
+    B1 = ROL64((A02^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A44^D4), 8);
+    A10 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A40^D0), 36);
+    B2 = ROL64((A11^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A03^D3), 56);
+    B0 = ROL64((A24^D4), 27);
+    A40 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A20^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A12^D2), 62);
+    B1 = ROL64((A33^D3), 55);
+    B2 = ROL64((A04^D4), 39);
+    A20 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A30^A10^A40^A20;
+    C1 = A21^A01^A31^A11^A41;
+    C2 = A42^A22^A02^A32^A12;
+    C3 = A13^A43^A23^A03^A33;
+    C4 = A34^A14^A44^A24^A04;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A01^D1), 44);
+    B2 = ROL64((A02^D2), 43);
+    B3 = ROL64((A03^D3), 21);
+    B4 = ROL64((A04^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+3];
+    A01 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A10^D0), 3);
+    B3 = ROL64((A11^D1), 45);
+    B4 = ROL64((A12^D2), 61);
+    B0 = ROL64((A13^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A10 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A20^D0), 18);
+    B0 = ROL64((A21^D1), 1);
+    B1 = ROL64((A22^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A24^D4), 8);
+    A20 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A30^D0), 36);
+    B2 = ROL64((A31^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A33^D3), 56);
+    B0 = ROL64((A34^D4), 27);
+    A30 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A40^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A42^D2), 62);
+    B1 = ROL64((A43^D3), 55);
+    B2 = ROL64((A44^D4), 39);
+    A40 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+  }
+}
+
+/*
+** Initialize a new hash.  iSize determines the size of the hash
+** in bits and should be one of 224, 256, 384, or 512.  Or iSize
+** can be zero to use the default hash size of 256 bits.
+*/
+static void SHA3Init(SHA3Context *p, int iSize){
+  memset(p, 0, sizeof(*p));
+  if( iSize>=128 && iSize<=512 ){
+    p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
+  }else{
+    p->nRate = (1600 - 2*256)/8;
+  }
+#if BYTEORDER==1234
+  /* Known to be little-endian at compile-time. No-op */
+#elif BYTEORDER==4321
+  p->ixMask = 7;  /* Big-endian */
+#else
+  {
+    static unsigned int one = 1;
+    if( 1==*(unsigned char*)&one ){
+      /* Little endian.  No byte swapping. */
+      p->ixMask = 0;
+    }else{
+      /* Big endian.  Byte swap. */
+      p->ixMask = 7;
+    }
+  }
+#endif
+}
+
+/*
+** Make consecutive calls to the SHA3Update function to add new content
+** to the hash
+*/
+static void SHA3Update(
+  SHA3Context *p,
+  const unsigned char *aData,
+  unsigned int nData
+){
+  unsigned int i = 0;
+#if BYTEORDER==1234
+  if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
+    for(; i+7<nData; i+=8){
+      p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
+      p->nLoaded += 8;
+      if( p->nLoaded>=p->nRate ){
+        KeccakF1600Step(p);
+        p->nLoaded = 0;
+      }
+    }
+  }
+#endif
+  for(; i<nData; i++){
+#if BYTEORDER==1234
+    p->u.x[p->nLoaded] ^= aData[i];
+#elif BYTEORDER==4321
+    p->u.x[p->nLoaded^0x07] ^= aData[i];
+#else
+    p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
+#endif
+    p->nLoaded++;
+    if( p->nLoaded==p->nRate ){
+      KeccakF1600Step(p);
+      p->nLoaded = 0;
+    }
+  }
+}
+
+/*
+** After all content has been added, invoke SHA3Final() to compute
+** the final hash.  The function returns a pointer to the binary
+** hash value.
+*/
+static unsigned char *SHA3Final(SHA3Context *p){
+  unsigned int i;
+  if( p->nLoaded==p->nRate-1 ){
+    const unsigned char c1 = 0x86;
+    SHA3Update(p, &c1, 1);
+  }else{
+    const unsigned char c2 = 0x06;
+    const unsigned char c3 = 0x80;
+    SHA3Update(p, &c2, 1);
+    p->nLoaded = p->nRate - 1;
+    SHA3Update(p, &c3, 1);
+  }
+  for(i=0; i<p->nRate; i++){
+    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
+  }
+  return &p->u.x[p->nRate];
+}
+
+/*
+** Convert a digest into base-16.  digest should be declared as
+** "unsigned char digest[20]" in the calling function.  The SHA3
+** digest is stored in the first 20 bytes.  zBuf should
+** be "char zBuf[41]".
+*/
+static void DigestToBase16(unsigned char *digest, char *zBuf, int nByte){
+  static const char zEncode[] = "0123456789abcdef";
+  int ix;
+
+  for(ix=0; ix<nByte; ix++){
+    *zBuf++ = zEncode[(*digest>>4)&0xf];
+    *zBuf++ = zEncode[*digest++ & 0xf];
+  }
+  *zBuf = '\0';
+}
+
+
+/*
+** Compute the SHA3 checksum of a file on disk.  Store the resulting
+** checksum in the blob pCksum.  pCksum is assumed to be initialized.
+**
+** Return the number of errors.
+*/
+static int sha3sum_file(const char *zFilename, int iSize, char *pCksum){
+  FILE *in;
+  SHA3Context ctx;
+  char zBuf[10240];
+
+  in = fopen(zFilename,"rb");
+  if( in==0 ){
+    return 1;
+  }
+  SHA3Init(&ctx, iSize);
+  for(;;){
+    int n = (int)fread(zBuf, 1, sizeof(zBuf), in);
+    if( n<=0 ) break;
+    SHA3Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
+  }
+  fclose(in);
+  DigestToBase16(SHA3Final(&ctx), pCksum, iSize/8);
+  return 0;
+}
+
+/*
+** The SHA1 implementation below is adapted from:
+**
+**  $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $
+**  $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $
+**
+** SHA-1 in C
+** By Steve Reid <steve@edmweb.com>
+** 100% Public Domain
+*/
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+static void SHA1Transform(
+  unsigned int state[5],
+  const unsigned char buffer[64]
+){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy context->state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/*
+ * SHA1Init - Initialize new context
+ */
+static void SHA1Init(SHA1Context *context){
+    /* SHA1 initialization constants */
+    context->state[0] = 0x67452301;
+    context->state[1] = 0xEFCDAB89;
+    context->state[2] = 0x98BADCFE;
+    context->state[3] = 0x10325476;
+    context->state[4] = 0xC3D2E1F0;
+    context->count[0] = context->count[1] = 0;
+}
+
+
+/*
+ * Run your data through this.
+ */
+static void SHA1Update(
+  SHA1Context *context,
+  const unsigned char *data,
+  unsigned int len
+){
+    unsigned int i, j;
+
+    j = context->count[0];
+    if ((context->count[0] += len << 3) < j)
+        context->count[1] += (len>>29)+1;
+    j = (j >> 3) & 63;
+    if ((j + len) > 63) {
+        (void)memcpy(&context->buffer[j], data, (i = 64-j));
+        SHA1Transform(context->state, context->buffer);
+        for ( ; i + 63 < len; i += 64)
+            SHA1Transform(context->state, &data[i]);
+        j = 0;
+    } else {
+        i = 0;
+    }
+    (void)memcpy(&context->buffer[j], &data[i], len - i);
+}
+
+
+/*
+ * Add padding and return the message digest.
+ */
+static void SHA1Final(unsigned char *digest, SHA1Context *context){
+    unsigned int i;
+    unsigned char finalcount[8];
+
+    for (i = 0; i < 8; i++) {
+        finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)]
+         >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+    }
+    SHA1Update(context, (const unsigned char *)"\200", 1);
+    while ((context->count[0] & 504) != 448)
+        SHA1Update(context, (const unsigned char *)"\0", 1);
+    SHA1Update(context, finalcount, 8);  /* Should cause a SHA1Transform() */
+
+    if (digest) {
+        for (i = 0; i < 20; i++)
+            digest[i] = (unsigned char)
+                ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+    }
+}
+
+
+/*
+** Compute the SHA1 checksum of a file on disk.  Store the resulting
+** checksum in the blob pCksum.  pCksum is assumed to be initialized.
+**
+** Return the number of errors.
+*/
+static int sha1sum_file(const char *zFilename, char *pCksum){
+  FILE *in;
+  SHA1Context ctx;
+  unsigned char zResult[20];
+  char zBuf[10240];
+
+  in = fopen(zFilename,"rb");
+  if( in==0 ){
+    return 1;
+  }
+  SHA1Init(&ctx);
+  for(;;){
+    int n = (int)fread(zBuf, 1, sizeof(zBuf), in);
+    if( n<=0 ) break;
+    SHA1Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
+  }
+  fclose(in);
+  SHA1Final(zResult, &ctx);
+  DigestToBase16(zResult, pCksum, 20);
+  return 0;
+}
+
+/*
+** Print a usage comment and quit.
+*/
+static void usage(const char *argv0){
+  fprintf(stderr, 
+     "Usage: %s manifest\n"
+     "Options:\n"
+     "   -v  Diagnostic output\n"
+     , argv0);
+  exit(1);
+}
+
+/*
+** Find the first whitespace character in a string.  Set that whitespace
+** to a \000 terminator and return a pointer to the next character.
+*/
+static char *nextToken(char *z){
+  while( *z && !isspace(*z) ) z++;
+  if( *z==0 ) return z;
+  *z = 0;
+  return &z[1];
+}
+  
+
+int main(int argc, char **argv){
+  const char *zManifest = 0;
+  int i;
+  int bVerbose = 0;
+  FILE *in;
+  int allValid = 1;
+  int rc;
+  char zDate[50];
+  char zHash[100];
+  char zLine[1000];
+
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' ){
+      if( z[1]=='-' ) z++;
+      if( strcmp(z, "-v")==0 ){
+        bVerbose = 1;
+      }else
+      {
+        fprintf(stderr, "unknown option \"%s\"", argv[i]);
+        exit(1);
+      }
+    }else if( zManifest!=0 ){
+      usage(argv[0]);
+    }else{
+      zManifest = z;
+    }
+  }
+  if( zManifest==0 ) usage(argv[0]);
+  zDate[0] = 0;
+  in = fopen(zManifest, "rb");
+  if( in==0 ){
+    fprintf(stderr, "cannot open \"%s\" for reading\n", zManifest);
+    exit(1);
+  }
+  while( fgets(zLine, sizeof(zLine), in) ){
+    if( strncmp(zLine, "D 20", 4)==0 ){
+      memcpy(zDate, &zLine[2], 10);
+      zDate[10] = ' ';
+      memcpy(&zDate[11], &zLine[13], 8);
+      zDate[19] = 0;
+      continue;
+    }
+    if( strncmp(zLine, "F ", 2)==0 ){
+      char *zFilename = &zLine[2];
+      char *zMHash = nextToken(zFilename);
+      nextToken(zMHash);
+      if( strlen(zMHash)==40 ){
+        rc = sha1sum_file(zFilename, zHash);
+      }else{
+        rc = sha3sum_file(zFilename, 256, zHash);
+      }
+      if( rc ){
+        allValid = 0;
+        if( bVerbose ){
+          printf("hash failed: %s\n", zFilename);
+        }else{
+          break;
+        }
+      }else if( strcmp(zHash, zMHash)!=0 ){
+        allValid = 0;
+        if( bVerbose ){
+          printf("wrong hash: %s\n", zFilename);
+          printf("... expected: %s\n", zMHash);
+          printf("... got:      %s\n", zHash);
+        }else{
+          break;
+        }
+      }
+    }
+  }
+  fclose(in);
+  sha3sum_file(zManifest, 256, zHash);
+  if( !allValid ){
+    printf("%s %.55s-modified\n", zDate, zHash);
+  }else{
+    printf("%s %s\n", zDate, zHash);
+  }
+  return 0;
+}
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index 5106a83857..ca40df8185 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -51,24 +51,9 @@ set zVersion [string trim [read $in]]
 close $in
 set nVersion [eval format "%d%03d%03d" [split $zVersion .]]
 
-# Get the fossil-scm version number from $TOP/manifest.uuid.
+# Get the source-id
 #
-set in [open $TOP/manifest.uuid]
-set zUuid [string trim [read $in]]
-close $in
-
-# Get the fossil-scm check-in date from the "D" card of $TOP/manifest.
-#
-set in [open $TOP/manifest]
-set zDate {}
-while {![eof $in]} {
-  set line [gets $in]
-  if {[regexp {^D (2[-0-9T:]+)} $line all date]} {
-    set zDate [string map {T { }} $date]
-    break
-  }
-}
-close $in
+set zSourceId [exec ./mksourceid $TOP/manifest]
 
 # Set up patterns for recognizing API declarations.
 #
@@ -125,7 +110,7 @@ foreach file $filelist {
 
     regsub -- --VERS--           $line $zVersion line
     regsub -- --VERSION-NUMBER-- $line $nVersion line
-    regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line
+    regsub -- --SOURCE-ID--      $line "$zSourceId" line
 
     if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} {
       set line "SQLITE_API $line"

From a1722547420bad6b4733ea1d82f7bc5f788129db Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 19:43:41 +0000
Subject: [PATCH 033/270] Attempting to fix the source-id generator so that it
 works for out-of-tree builds.

FossilOrigin-Name: 5a037ac2da7449be3e26c36910ac5d865d7e74d3c25af0a10578c1f92fe2afea
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/mksqlite3h.tcl |  5 ++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 6f9319811f..75edf80b05 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"mksourceid"\sprogram\sto\sthe\sbuild\sprocess.\s\sThat\sprogram\schanges\nthe\sSQLITE_SOURCE_ID\sif\sthe\ssource\stree\shas\sbeen\smodified\sin\sany\sway.
-D 2017-08-22T19:19:00.280
+C Attempting\sto\sfix\sthe\ssource-id\sgenerator\sso\sthat\sit\sworks\sfor\sout-of-tree\nbuilds.
+D 2017-08-22T19:43:41.092
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -1592,7 +1592,7 @@ F tool/mksourceid.c 1db4636bf4249ac5d2d56fb06b0b30929bf7446a88c7048bb731a72a1c47
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
 F tool/mksqlite3c.tcl a4b36eaa002ed00a0ab2c93d999a14f1acae98ff09a85382e5abc05a91edb82b
-F tool/mksqlite3h.tcl e10b7878ca20161594fbc28913027956d7ee35713400534a115d8f139b496478
+F tool/mksqlite3h.tcl 5dbb75b1125000907f5b334b7b7a16960c5805a26c7a7475f6dd9bd415709a71
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254
-R 48a68cf4cf699ac46128e2797c5425f9
+P d4c05e04f7e1325a3260808ee17252876f678e78bf0cf6569a18a52ff674bd7a
+R cac9e20accccd833acf3f1836f6cc4f1
 U drh
-Z 40e98cab2922888e2c30394283edf487
+Z 8bf43d2286864b6e544f292f098efb98
diff --git a/manifest.uuid b/manifest.uuid
index 616786c10c..aa5c13477d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4c05e04f7e1325a3260808ee17252876f678e78bf0cf6569a18a52ff674bd7a
\ No newline at end of file
+5a037ac2da7449be3e26c36910ac5d865d7e74d3c25af0a10578c1f92fe2afea
\ No newline at end of file
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index ca40df8185..174a3333bf 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -53,7 +53,10 @@ set nVersion [eval format "%d%03d%03d" [split $zVersion .]]
 
 # Get the source-id
 #
-set zSourceId [exec ./mksourceid $TOP/manifest]
+set PWD [pwd]
+cd $TOP
+set zSourceId [exec ./mksourceid manifest]
+cd $PWD
 
 # Set up patterns for recognizing API declarations.
 #

From c9aed7f891242dbd3c8d046986368d35bb23f656 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 19:49:34 +0000
Subject: [PATCH 034/270] Trying again to get out-of-tree builds to work
 correctly.

FossilOrigin-Name: a1b3337e949fc431e19a3d977d07a312bb253ab7fec6811c0221abd514985d55
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/mksqlite3h.tcl |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 75edf80b05..231c05503d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Attempting\sto\sfix\sthe\ssource-id\sgenerator\sso\sthat\sit\sworks\sfor\sout-of-tree\nbuilds.
-D 2017-08-22T19:43:41.092
+C Trying\sagain\sto\sget\sout-of-tree\sbuilds\sto\swork\scorrectly.
+D 2017-08-22T19:49:34.691
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -1592,7 +1592,7 @@ F tool/mksourceid.c 1db4636bf4249ac5d2d56fb06b0b30929bf7446a88c7048bb731a72a1c47
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
 F tool/mksqlite3c.tcl a4b36eaa002ed00a0ab2c93d999a14f1acae98ff09a85382e5abc05a91edb82b
-F tool/mksqlite3h.tcl 5dbb75b1125000907f5b334b7b7a16960c5805a26c7a7475f6dd9bd415709a71
+F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d4c05e04f7e1325a3260808ee17252876f678e78bf0cf6569a18a52ff674bd7a
-R cac9e20accccd833acf3f1836f6cc4f1
+P 5a037ac2da7449be3e26c36910ac5d865d7e74d3c25af0a10578c1f92fe2afea
+R 5f7cd0b5ad89a4d7783875a0ce64d27b
 U drh
-Z 8bf43d2286864b6e544f292f098efb98
+Z ce17968a3f11b573ce5537607e65c92d
diff --git a/manifest.uuid b/manifest.uuid
index aa5c13477d..cfd5e7f801 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5a037ac2da7449be3e26c36910ac5d865d7e74d3c25af0a10578c1f92fe2afea
\ No newline at end of file
+a1b3337e949fc431e19a3d977d07a312bb253ab7fec6811c0221abd514985d55
\ No newline at end of file
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index 174a3333bf..5b4c48bb74 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -55,7 +55,7 @@ set nVersion [eval format "%d%03d%03d" [split $zVersion .]]
 #
 set PWD [pwd]
 cd $TOP
-set zSourceId [exec ./mksourceid manifest]
+set zSourceId [exec $PWD/mksourceid manifest]
 cd $PWD
 
 # Set up patterns for recognizing API declarations.

From 0a02c72e7914eb1bcd6d2d930ec28199bd1839ae Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 21:07:03 +0000
Subject: [PATCH 035/270] Less dramatic changes to the source-id following an
 edit.  Modify the way that the amalgamation is constructed to give it the
 opportunity to detect changes and modify the source-id.

FossilOrigin-Name: 564c7340a3368501c3da885afde52123ed7f558801f6190cbe6173dfe9704b70
---
 manifest            | 17 ++++++++---------
 manifest.uuid       |  2 +-
 src/shell.c         |  2 +-
 tool/mksourceid.c   |  4 ++--
 tool/mksqlite3c.tcl | 38 +++++++++++++++++++++++++++++++++++++-
 5 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 1c80411530..4ce365fa8d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\sSQLITE_SOURCE_ID\sif\sthe\ssource\scode\shas\schanged\sin\sany\sway\ssince\nthe\sprevious\scheck-in.
-D 2017-08-22T19:54:34.821
+C Less\sdramatic\schanges\sto\sthe\ssource-id\sfollowing\san\sedit.\s\sModify\sthe\sway\nthat\sthe\samalgamation\sis\sconstructed\sto\sgive\sit\sthe\sopportunity\sto\sdetect\nchanges\sand\smodify\sthe\ssource-id.
+D 2017-08-22T21:07:03.663
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -457,7 +457,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
-F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
+F src/shell.c 8f2a8b9e4ffe4f4596b1690dd628cd355d5605257e14ddba83daf5422e0e39af
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
 F src/sqlite.h.in f0bd1abf5e27bd22b3bcaae2a861c1efc4ab7e752bf7eb102355135617eb8199
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1588,10 +1588,10 @@ F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
 F tool/mkshellc.tcl 69c38ecd7b74b2b0799a35ce20e1e3998e504d8c99c100ca4b98ae9d8f6279bc
-F tool/mksourceid.c 1db4636bf4249ac5d2d56fb06b0b30929bf7446a88c7048bb731a72a1c4767ea
+F tool/mksourceid.c 30966d568654a4fd962fb324753e49429b7379e1f72d2be489ade963121f5943
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
-F tool/mksqlite3c.tcl a4b36eaa002ed00a0ab2c93d999a14f1acae98ff09a85382e5abc05a91edb82b
+F tool/mksqlite3c.tcl b258d679829a9305f5cf107b7d97b9bf23adb3773df42947fed5ef7b180dfbd9
 F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
@@ -1650,8 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 885c2b44a44f8d054014e4079b2cac8279c11d13206d5b5215189ef75b9c5254 a1b3337e949fc431e19a3d977d07a312bb253ab7fec6811c0221abd514985d55
-R 5f7cd0b5ad89a4d7783875a0ce64d27b
-T +closed a1b3337e949fc431e19a3d977d07a312bb253ab7fec6811c0221abd514985d55
+P 515d6a8377cc1dc76d2e78e242fe256cbeef1c1217ec35367648ddeeb17007ec
+R a4db1e4547b3dd92436518e89aa3b90a
 U drh
-Z d2c0063b0c5ab8271ba1b5751cfab094
+Z 98b8ca1d9a43edd06d77198cb0511ee0
diff --git a/manifest.uuid b/manifest.uuid
index e1681bb498..92dd519d2e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-515d6a8377cc1dc76d2e78e242fe256cbeef1c1217ec35367648ddeeb17007ec
\ No newline at end of file
+564c7340a3368501c3da885afde52123ed7f558801f6190cbe6173dfe9704b70
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index eaefe62012..bd7853f65d 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -8023,7 +8023,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   stdout_is_console = isatty(1);
 
 #if USE_SYSTEM_SQLITE+0!=1
-  if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){
+  if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
     utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
             sqlite3_sourceid(), SQLITE_SOURCE_ID);
     exit(1);
diff --git a/tool/mksourceid.c b/tool/mksourceid.c
index 722728be4c..0e20b3c3d3 100644
--- a/tool/mksourceid.c
+++ b/tool/mksourceid.c
@@ -7,7 +7,7 @@
 **      SHA3 hash of the manifest file.
 **
 ** (2)  All individual file hashes in the manifest are verified.  If any
-**      source file has changed, the SHA3 hash ends with "-modified".
+**      source file has changed, the SHA3 hash ends with "modified".
 **
 */
 #include <stdlib.h>
@@ -844,7 +844,7 @@ int main(int argc, char **argv){
   fclose(in);
   sha3sum_file(zManifest, 256, zHash);
   if( !allValid ){
-    printf("%s %.55s-modified\n", zDate, zHash);
+    printf("%s %.60salt1\n", zDate, zHash);
   }else{
     printf("%s %s\n", zDate, zHash);
   }
diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl
index 3c3f2c35a0..933819d12b 100644
--- a/tool/mksqlite3c.tcl
+++ b/tool/mksqlite3c.tcl
@@ -242,7 +242,13 @@ proc copy_file {filename} {
             }
           }
           append line $funcname $rest
-          puts $out $line
+          if {$funcname=="sqlite3_sourceid" && !$linemacros} {
+            # The sqlite3_sourceid() routine is synthesized at the end of
+            # the amalgamation
+            puts $out "/* $line */"
+          } else {
+            puts $out $line
+          }
         } else {
           puts $out "SQLITE_PRIVATE $line"
         }
@@ -396,4 +402,34 @@ foreach file {
   copy_file tsrc/$file
 }
 
+# Synthesize an alternative sqlite3_sourceid() implementation that
+# that tries to detects changes in the amalgamation source text
+# and modify returns a modified source-id if changes are detected.
+#
+# The only detection mechanism we have is the __LINE__ macro.  So only
+# edits that changes the number of lines of source code are detected.
+#
+if {!$linemacros} {
+  flush $out
+  set in2 [open sqlite3.c]
+  set cnt 0
+  set oldsrcid {}
+  while {![eof $in2]} {
+    incr cnt
+    gets $in2 line
+    if {[regexp {^#define SQLITE_SOURCE_ID } $line]} {set oldsrcid $line}
+  }
+  close $in2
+  regsub {[0-9a-flt]{4}"} $oldsrcid {alt2"} oldsrcid
+  puts $out \
+"#if __LINE__!=[expr {$cnt+0}]
+#undef SQLITE_SOURCE_ID
+$oldsrcid
+#endif
+/* Return the source-id for this library */
+SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }"
+}
+puts $out \
+"/************************** End of sqlite3.c ******************************/"
+
 close $out

From 489a224bea393ab9a661d02d693dfea70a97d599 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 22 Aug 2017 21:23:02 +0000
Subject: [PATCH 036/270] Update documentation to make it clear that
 SQLITE_SOURCE_ID and sqlite3_sourceid() might changes if the source code is
 edited.

FossilOrigin-Name: e69c0c8770d3cc2fcdc779c6290caf1575644d457326caa00327b9070251d620
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 12 ++++++++----
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 4ce365fa8d..55217ab212 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Less\sdramatic\schanges\sto\sthe\ssource-id\sfollowing\san\sedit.\s\sModify\sthe\sway\nthat\sthe\samalgamation\sis\sconstructed\sto\sgive\sit\sthe\sopportunity\sto\sdetect\nchanges\sand\smodify\sthe\ssource-id.
-D 2017-08-22T21:07:03.663
+C Update\sdocumentation\sto\smake\sit\sclear\sthat\sSQLITE_SOURCE_ID\sand\nsqlite3_sourceid()\smight\schanges\sif\sthe\ssource\scode\sis\sedited.
+D 2017-08-22T21:23:02.258
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -459,7 +459,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c 8f2a8b9e4ffe4f4596b1690dd628cd355d5605257e14ddba83daf5422e0e39af
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
-F src/sqlite.h.in f0bd1abf5e27bd22b3bcaae2a861c1efc4ab7e752bf7eb102355135617eb8199
+F src/sqlite.h.in 73a75cc53b5017525332ff3db2f74bb2607f1e8c0e8974d4afa3df3786f70aab
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h bc0792db4ff887e4884d386188527e1cd7c16d496c8f3ba23333e68cee3e4f78
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 515d6a8377cc1dc76d2e78e242fe256cbeef1c1217ec35367648ddeeb17007ec
-R a4db1e4547b3dd92436518e89aa3b90a
+P 564c7340a3368501c3da885afde52123ed7f558801f6190cbe6173dfe9704b70
+R b1a5f4a03931a03b3cf49a93760947b3
 U drh
-Z 98b8ca1d9a43edd06d77198cb0511ee0
+Z 80eecc829d5290754bef086590571027
diff --git a/manifest.uuid b/manifest.uuid
index 92dd519d2e..0e97d0f48f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-564c7340a3368501c3da885afde52123ed7f558801f6190cbe6173dfe9704b70
\ No newline at end of file
+e69c0c8770d3cc2fcdc779c6290caf1575644d457326caa00327b9070251d620
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 1020e5f3d6..96d133b41c 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -115,7 +115,9 @@ extern "C" {
 ** a string which identifies a particular check-in of SQLite
 ** within its configuration management system.  ^The SQLITE_SOURCE_ID
 ** string contains the date and time of the check-in (UTC) and a SHA1
-** or SHA3-256 hash of the entire source tree.
+** or SHA3-256 hash of the entire source tree.  If the source code has
+** been edited in any way since it was last checked in, then the last
+** four hexadecimal digits of the hash may be modified.
 **
 ** See also: [sqlite3_libversion()],
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
@@ -139,7 +141,7 @@ extern "C" {
 **
 ** <blockquote><pre>
 ** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
-** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
 ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
 ** </pre></blockquote>)^
 **
@@ -149,9 +151,11 @@ extern "C" {
 ** function is provided for use in DLLs since DLL users usually do not have
 ** direct access to string constants within the DLL.  ^The
 ** sqlite3_libversion_number() function returns an integer equal to
-** [SQLITE_VERSION_NUMBER].  ^The sqlite3_sourceid() function returns 
+** [SQLITE_VERSION_NUMBER].  ^(The sqlite3_sourceid() function returns 
 ** a pointer to a string constant whose value is the same as the 
-** [SQLITE_SOURCE_ID] C preprocessor macro.
+** [SQLITE_SOURCE_ID] C preprocessor macro.  Except if SQLite is built
+** using an edited copy of [the amalgamation], then the last four characters
+** of the hash might be different from [SQLITE_SOURCE_ID].)^
 **
 ** See also: [sqlite_version()] and [sqlite_source_id()].
 */

From 5e27e1dc499097272615b0591c457b7967aedf24 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 23 Aug 2017 14:45:59 +0000
Subject: [PATCH 037/270] Smaller and faster implementation of the fillInCell()
 routine.

FossilOrigin-Name: 77074990edef6c42bdfe7ccce1affc8ed64a86dde3ad4fa50b60ba0a6d851eea
---
 manifest      | 12 +++----
 manifest.uuid |  2 +-
 src/btree.c   | 99 +++++++++++++++++++++++++++++----------------------
 3 files changed, 63 insertions(+), 50 deletions(-)

diff --git a/manifest b/manifest
index 55217ab212..79af87c73a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sdocumentation\sto\smake\sit\sclear\sthat\sSQLITE_SOURCE_ID\sand\nsqlite3_sourceid()\smight\schanges\sif\sthe\ssource\scode\sis\sedited.
-D 2017-08-22T21:23:02.258
+C Smaller\sand\sfaster\simplementation\sof\sthe\sfillInCell()\sroutine.
+D 2017-08-23T14:45:59.667
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c e1a98bef27d6189df5c5b8b3c9b5e53d6b74bf9a936af97757c21e9ea6a5c6b2
+F src/btree.c ea10ca63430533c358da32e5d2b8b22aa3c6a9e84f295139e4e5bb00f635726d
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 564c7340a3368501c3da885afde52123ed7f558801f6190cbe6173dfe9704b70
-R b1a5f4a03931a03b3cf49a93760947b3
+P e69c0c8770d3cc2fcdc779c6290caf1575644d457326caa00327b9070251d620
+R 66591a7ad128e50f7a0c04de7cb52832
 U drh
-Z 80eecc829d5290754bef086590571027
+Z 50b9a412665b926c36fcdf17b0a625f8
diff --git a/manifest.uuid b/manifest.uuid
index 0e97d0f48f..0e5f5791b5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e69c0c8770d3cc2fcdc779c6290caf1575644d457326caa00327b9070251d620
\ No newline at end of file
+77074990edef6c42bdfe7ccce1affc8ed64a86dde3ad4fa50b60ba0a6d851eea
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index ecdbb6d0a3..b6e5fbc508 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -6218,21 +6218,20 @@ static int fillInCell(
 ){
   int nPayload;
   const u8 *pSrc;
-  int nSrc, n, rc;
+  int nSrc, n, rc, mn;
   int spaceLeft;
-  MemPage *pOvfl = 0;
-  MemPage *pToRelease = 0;
+  MemPage *pToRelease;
   unsigned char *pPrior;
   unsigned char *pPayload;
-  BtShared *pBt = pPage->pBt;
-  Pgno pgnoOvfl = 0;
+  BtShared *pBt;
+  Pgno pgnoOvfl;
   int nHeader;
 
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
 
   /* pPage is not necessarily writeable since pCell might be auxiliary
   ** buffer space that is separate from the pPage buffer area */
-  assert( pCell<pPage->aData || pCell>=&pPage->aData[pBt->pageSize]
+  assert( pCell<pPage->aData || pCell>=&pPage->aData[pPage->pBt->pageSize]
             || sqlite3PagerIswriteable(pPage->pDbPage) );
 
   /* Fill in the header. */
@@ -6252,25 +6251,36 @@ static int fillInCell(
   }
   
   /* Fill in the payload */
+  pPayload = &pCell[nHeader];
   if( nPayload<=pPage->maxLocal ){
+    /* This is the common case where everything fits on the btree page
+    ** and no overflow pages are required. */
     n = nHeader + nPayload;
     testcase( n==3 );
     testcase( n==4 );
     if( n<4 ) n = 4;
     *pnSize = n;
-    spaceLeft = nPayload;
-    pPrior = pCell;
-  }else{
-    int mn = pPage->minLocal;
-    n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4);
-    testcase( n==pPage->maxLocal );
-    testcase( n==pPage->maxLocal+1 );
-    if( n > pPage->maxLocal ) n = mn;
-    spaceLeft = n;
-    *pnSize = n + nHeader + 4;
-    pPrior = &pCell[nHeader+n];
+    assert( nSrc<=nPayload );
+    testcase( nSrc<nPayload );
+    memcpy(pPayload, pSrc, nSrc);
+    memset(pPayload+nSrc, 0, nPayload-nSrc);
+    return SQLITE_OK;
   }
-  pPayload = &pCell[nHeader];
+
+  /* If we reach this point, it means that some of the content will need
+  ** to spill onto overflow pages.
+  */
+  mn = pPage->minLocal;
+  n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4);
+  testcase( n==pPage->maxLocal );
+  testcase( n==pPage->maxLocal+1 );
+  if( n > pPage->maxLocal ) n = mn;
+  spaceLeft = n;
+  *pnSize = n + nHeader + 4;
+  pPrior = &pCell[nHeader+n];
+  pToRelease = 0;
+  pgnoOvfl = 0;
+  pBt = pPage->pBt;
 
   /* At this point variables should be set as follows:
   **
@@ -6296,8 +6306,35 @@ static int fillInCell(
 #endif
 
   /* Write the payload into the local Cell and any extra into overflow pages */
-  while( nPayload>0 ){
+  while( 1 ){
+    n = nPayload;
+    if( n>spaceLeft ) n = spaceLeft;
+
+    /* If pToRelease is not zero than pPayload points into the data area
+    ** of pToRelease.  Make sure pToRelease is still writeable. */
+    assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) );
+
+    /* If pPayload is part of the data area of pPage, then make sure pPage
+    ** is still writeable */
+    assert( pPayload<pPage->aData || pPayload>=&pPage->aData[pBt->pageSize]
+            || sqlite3PagerIswriteable(pPage->pDbPage) );
+
+    if( nSrc>=n ){
+      memcpy(pPayload, pSrc, n);
+    }else if( nSrc>0 ){
+      n = nSrc;
+      memcpy(pPayload, pSrc, n);
+    }else{
+      memset(pPayload, 0, n);
+    }
+    nPayload -= n;
+    if( nPayload<=0 ) break;
+    pPayload += n;
+    pSrc += n;
+    nSrc -= n;
+    spaceLeft -= n;
     if( spaceLeft==0 ){
+      MemPage *pOvfl = 0;
 #ifndef SQLITE_OMIT_AUTOVACUUM
       Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */
       if( pBt->autoVacuum ){
@@ -6350,30 +6387,6 @@ static int fillInCell(
       pPayload = &pOvfl->aData[4];
       spaceLeft = pBt->usableSize - 4;
     }
-    n = nPayload;
-    if( n>spaceLeft ) n = spaceLeft;
-
-    /* If pToRelease is not zero than pPayload points into the data area
-    ** of pToRelease.  Make sure pToRelease is still writeable. */
-    assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) );
-
-    /* If pPayload is part of the data area of pPage, then make sure pPage
-    ** is still writeable */
-    assert( pPayload<pPage->aData || pPayload>=&pPage->aData[pBt->pageSize]
-            || sqlite3PagerIswriteable(pPage->pDbPage) );
-
-    if( nSrc>0 ){
-      if( n>nSrc ) n = nSrc;
-      assert( pSrc );
-      memcpy(pPayload, pSrc, n);
-    }else{
-      memset(pPayload, 0, n);
-    }
-    nPayload -= n;
-    pPayload += n;
-    pSrc += n;
-    nSrc -= n;
-    spaceLeft -= n;
   }
   releasePage(pToRelease);
   return SQLITE_OK;

From 5e398e4cbd2d3f05903bbb7606c4bb15d9c122b2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 23 Aug 2017 20:36:06 +0000
Subject: [PATCH 038/270] Size and performance optimization to dropCell() and
 freeSpace().

FossilOrigin-Name: bc1ec123ce05c9d16b0942f870381145dc9725764e47806939ff207a73066f4d
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   | 32 +++++++++++++++++---------------
 3 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index 79af87c73a..80d60559cb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Smaller\sand\sfaster\simplementation\sof\sthe\sfillInCell()\sroutine.
-D 2017-08-23T14:45:59.667
+C Size\sand\sperformance\soptimization\sto\sdropCell()\sand\sfreeSpace().
+D 2017-08-23T20:36:06.554
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c ea10ca63430533c358da32e5d2b8b22aa3c6a9e84f295139e4e5bb00f635726d
+F src/btree.c 45f3b8b3ad112d21afacfdfcbc1c362637bf080abb58c901104a8d3a6207a1bf
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e69c0c8770d3cc2fcdc779c6290caf1575644d457326caa00327b9070251d620
-R 66591a7ad128e50f7a0c04de7cb52832
+P 77074990edef6c42bdfe7ccce1affc8ed64a86dde3ad4fa50b60ba0a6d851eea
+R 522ed57ec99b8b2e95cc60b01e62a8eb
 U drh
-Z 50b9a412665b926c36fcdf17b0a625f8
+Z 185127068afaca8c598b7b9a50979c09
diff --git a/manifest.uuid b/manifest.uuid
index 0e5f5791b5..07ad7f8eec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-77074990edef6c42bdfe7ccce1affc8ed64a86dde3ad4fa50b60ba0a6d851eea
\ No newline at end of file
+bc1ec123ce05c9d16b0942f870381145dc9725764e47806939ff207a73066f4d
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index b6e5fbc508..bd66a77049 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -1505,7 +1505,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
     if( (x = size - nByte)>=0 ){
       testcase( x==4 );
       testcase( x==3 );
-      if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){
+      if( size+pc > usableSize ){
         *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
         return 0;
       }else if( x<4 ){
@@ -1640,7 +1640,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   u8 hdr;                               /* Page header size.  0 or 100 */
   u8 nFrag = 0;                         /* Reduction in fragmentation */
   u16 iOrigSize = iSize;                /* Original value of iSize */
-  u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */
+  u16 x;                                /* Offset to cell content area */
   u32 iEnd = iStart + iSize;            /* First byte past the iStart buffer */
   unsigned char *data = pPage->aData;   /* Page content */
 
@@ -1650,13 +1650,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize );
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   assert( iSize>=4 );   /* Minimum cell size is 4 */
-  assert( iStart<=iLast );
-
-  /* Overwrite deleted information with zeros when the secure_delete
-  ** option is enabled */
-  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
-    memset(&data[iStart], 0, iSize);
-  }
+  assert( iStart<=pPage->pBt->usableSize-4 );
 
   /* The list of freeblocks must be in ascending order.  Find the 
   ** spot on the list where iStart should be inserted.
@@ -1673,7 +1667,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
       }
       iPtr = iFreeBlk;
     }
-    if( iFreeBlk>iLast ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    if( iFreeBlk>pPage->pBt->usableSize-4 ){
+      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    }
     assert( iFreeBlk>iPtr || iFreeBlk==0 );
   
     /* At this point:
@@ -1709,19 +1705,25 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
     data[hdr+7] -= nFrag;
   }
-  if( iStart==get2byte(&data[hdr+5]) ){
+  x = get2byte(&data[hdr+5]);
+  if( iStart<=x ){
     /* The new freeblock is at the beginning of the cell content area,
     ** so just extend the cell content area rather than create another
     ** freelist entry */
-    if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    if( iStart<x || iPtr!=hdr+1 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
     put2byte(&data[hdr+1], iFreeBlk);
     put2byte(&data[hdr+5], iEnd);
   }else{
     /* Insert the new freeblock into the freelist */
     put2byte(&data[iPtr], iStart);
-    put2byte(&data[iStart], iFreeBlk);
-    put2byte(&data[iStart+2], iSize);
   }
+  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
+    /* Overwrite deleted information with zeros when the secure_delete
+    ** option is enabled */
+    memset(&data[iStart], 0, iSize);
+  }
+  put2byte(&data[iStart], iFreeBlk);
+  put2byte(&data[iStart+2], iSize);
   pPage->nFree += iOrigSize;
   return SQLITE_OK;
 }
@@ -6418,7 +6420,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){
   hdr = pPage->hdrOffset;
   testcase( pc==get2byte(&data[hdr+5]) );
   testcase( pc+sz==pPage->pBt->usableSize );
-  if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){
+  if( pc+sz > pPage->pBt->usableSize ){
     *pRC = SQLITE_CORRUPT_BKPT;
     return;
   }

From 87d63c900db707562a0a1fde30eb98e69e33b7a1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 23 Aug 2017 23:09:03 +0000
Subject: [PATCH 039/270] Performance optimization to pageFindSlot() in the
 b-tree layer.

FossilOrigin-Name: 59560d079fab4b91ec50855cc60349da178209c38fb6dae674ff874ccfc7f5fa
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   | 16 +++++++---------
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 80d60559cb..9a9edb7494 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\sand\sperformance\soptimization\sto\sdropCell()\sand\sfreeSpace().
-D 2017-08-23T20:36:06.554
+C Performance\soptimization\sto\spageFindSlot()\sin\sthe\sb-tree\slayer.
+D 2017-08-23T23:09:03.990
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 45f3b8b3ad112d21afacfdfcbc1c362637bf080abb58c901104a8d3a6207a1bf
+F src/btree.c f93c05dbd60551b322dbcaaf4961e87ea20eb67667ebc3181f15d2b5cb66bbaa
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 77074990edef6c42bdfe7ccce1affc8ed64a86dde3ad4fa50b60ba0a6d851eea
-R 522ed57ec99b8b2e95cc60b01e62a8eb
+P bc1ec123ce05c9d16b0942f870381145dc9725764e47806939ff207a73066f4d
+R f6b2325c9199ea50423580e65f552033
 U drh
-Z 185127068afaca8c598b7b9a50979c09
+Z 8c1fbb63bd6c44847b2e56eee310d2d1
diff --git a/manifest.uuid b/manifest.uuid
index 07ad7f8eec..7fb20a017d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bc1ec123ce05c9d16b0942f870381145dc9725764e47806939ff207a73066f4d
\ No newline at end of file
+59560d079fab4b91ec50855cc60349da178209c38fb6dae674ff874ccfc7f5fa
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index bd66a77049..a4bcaf6c49 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -1488,16 +1488,10 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
   int pc = get2byte(&aData[iAddr]);
   int x;
   int usableSize = pPg->pBt->usableSize;
+  int size;            /* Size of the free slot */
 
   assert( pc>0 );
-  do{
-    int size;            /* Size of the free slot */
-    /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
-    ** increasing offset. */
-    if( pc>usableSize-4 || pc<iAddr+4 ){
-      *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
-      return 0;
-    }
+  while( pc<=usableSize-4 ){
     /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each
     ** freeblock form a big-endian integer which is the size of the freeblock
     ** in bytes, including the 4-byte header. */
@@ -1526,7 +1520,11 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
     }
     iAddr = pc;
     pc = get2byte(&aData[pc]);
-  }while( pc );
+    if( pc<iAddr+size ) break;
+  }
+  if( pc ){
+    *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
+  }
 
   return 0;
 }

From 9c54156a9cc203a106f9938eab1a572c7f3598f3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 24 Aug 2017 10:10:28 +0000
Subject: [PATCH 040/270] Test BEGIN CONCURRENT transactions that consist
 entirely of read-only statements.

FossilOrigin-Name: c3fe1f4b7e8dcadcb516622719d000b808effe3ad497244ba44f57d52dc2cc08
---
 manifest              | 12 +++---
 manifest.uuid         |  2 +-
 test/concurrent4.test | 89 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 91cd200ac0..c5496f5966 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\smore\srigorous\stest\scase\sfor\sthe\sbug\sfixed\sby\sthe\sprevious\scommit\son\sthis\nbranch.
-D 2017-08-12T14:06:15.050
+C Test\sBEGIN\sCONCURRENT\stransactions\sthat\sconsist\sentirely\sof\sread-only\nstatements.
+D 2017-08-24T10:10:28.736
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -665,7 +665,7 @@ F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43
 F test/concurrent.test a801cd60c370f0ed851657c9576b102f9ab1dd846c6a88e6ae45939a8deeda7c
 F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8
 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170
-F test/concurrent4.test 653de3066911acfb9dcf3802bf4f1981b392b86c11f75e2c38ed1abfdd162293
+F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44
 F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
@@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38dd9b50fe260d853cbc2551bc1bb60ddf5752f0456e0da3afe4cbf728c891d8
-R 5ae6eec944d8695060377eb5fcb02f25
+P 4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920
+R 03a159c270fd82b6d54f5498b523d32e
 U dan
-Z be5258dc78cb84266e8cf9b3ea9ebd92
+Z fcb6179847dc0180b01d1fb922b3b8b6
diff --git a/manifest.uuid b/manifest.uuid
index d2eec08d49..7d481eeeee 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4256072399f44f48ed0856aa8112226af6feaf8676923612bde6cea239ebf920
\ No newline at end of file
+c3fe1f4b7e8dcadcb516622719d000b808effe3ad497244ba44f57d52dc2cc08
\ No newline at end of file
diff --git a/test/concurrent4.test b/test/concurrent4.test
index dd29434381..d43e701d99 100644
--- a/test/concurrent4.test
+++ b/test/concurrent4.test
@@ -101,5 +101,94 @@ do_execsql_test 3.2 {
   COMMIT;
 } {}
 
+#-------------------------------------------------------------------------
+# Test the effect of BEGIN CONCURRENT transactions that consist entirely
+# of read-only statements.
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES(1, 2);
+  INSERT INTO t4 VALUES(3, 4);
+} {wal}
+ 
+sqlite3 db2 test.db
+do_test 4.1.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(5, 6);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    ROLLBACK;
+  }
+} {1 2 3 4}
+
+do_test 4.1.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6}
+
+do_test 4.2.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(7, 8);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6}
+
+do_test 4.2.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6 7 8}
+
+do_test 4.3 {
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+  }
+
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(9, 10);
+    COMMIT;
+  }
+  db2 eval {
+    SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6 7 8}
+
+set sz [file size test.db-wal]
+do_test 4.4.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+  }
+
+  db eval COMMIT
+  file size test.db-wal
+} $sz
+do_test 4.4.2 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+    ROLLBACK;
+  }
+  file size test.db-wal
+} $sz
+
 finish_test
 

From 526740b1b91820e4e0944dd9905be91cec281f12 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 24 Aug 2017 13:55:46 +0000
Subject: [PATCH 041/270] Make sure the sqlite3_result_pointer() interface does
 not leave a VM register in an inconsistent state.  Fix for ticket
 [7486aa54b968e9b5].  Test cases are in TH3.

FossilOrigin-Name: d2f9230c5c7ad6166e7d2b649f77960fa58b1cb583e529a43882753ab348413c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbeapi.c |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 9a9edb7494..0da8256bdb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Performance\soptimization\sto\spageFindSlot()\sin\sthe\sb-tree\slayer.
-D 2017-08-23T23:09:03.990
+C Make\ssure\sthe\ssqlite3_result_pointer()\sinterface\sdoes\snot\sleave\sa\sVM\sregister\nin\san\sinconsistent\sstate.\s\sFix\sfor\sticket\s[7486aa54b968e9b5].\s\sTest\scases\nare\sin\sTH3.
+D 2017-08-24T13:55:46.912
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -527,7 +527,7 @@ F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
-F src/vdbeapi.c 05d6b14ab73952db0d73f6452d6960216997bd966a710266b2fe051f25326abc
+F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bc1ec123ce05c9d16b0942f870381145dc9725764e47806939ff207a73066f4d
-R f6b2325c9199ea50423580e65f552033
+P 59560d079fab4b91ec50855cc60349da178209c38fb6dae674ff874ccfc7f5fa
+R f928c1195d839713ee78e6b46c1884f1
 U drh
-Z 8c1fbb63bd6c44847b2e56eee310d2d1
+Z fae59aff59c16e3b9c2ad1cd0f0e8fa3
diff --git a/manifest.uuid b/manifest.uuid
index 7fb20a017d..c161e3d95e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-59560d079fab4b91ec50855cc60349da178209c38fb6dae674ff874ccfc7f5fa
\ No newline at end of file
+d2f9230c5c7ad6166e7d2b649f77960fa58b1cb583e529a43882753ab348413c
\ No newline at end of file
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 3f4ef8f6a9..b9df40b8fd 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -398,7 +398,8 @@ void sqlite3_result_pointer(
 ){
   Mem *pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pOut->db->mutex) );
-  sqlite3VdbeMemSetNull(pOut);
+  sqlite3VdbeMemRelease(pOut);
+  pOut->flags = MEM_Null;
   sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor);
 }
 void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){

From 09e16491f603e4cdfa277d48a0989ba79ba4e396 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 24 Aug 2017 15:43:26 +0000
Subject: [PATCH 042/270] Fixes to documentation about SQLITE_OPEN_URI.

FossilOrigin-Name: 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 0da8256bdb..a5acff37b1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\ssqlite3_result_pointer()\sinterface\sdoes\snot\sleave\sa\sVM\sregister\nin\san\sinconsistent\sstate.\s\sFix\sfor\sticket\s[7486aa54b968e9b5].\s\sTest\scases\nare\sin\sTH3.
-D 2017-08-24T13:55:46.912
+C Fixes\sto\sdocumentation\sabout\sSQLITE_OPEN_URI.
+D 2017-08-24T15:43:26.350
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -459,7 +459,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c 8f2a8b9e4ffe4f4596b1690dd628cd355d5605257e14ddba83daf5422e0e39af
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
-F src/sqlite.h.in 73a75cc53b5017525332ff3db2f74bb2607f1e8c0e8974d4afa3df3786f70aab
+F src/sqlite.h.in a8e60396a73996a12a153299943f45fe59202c89bb1a46bab203a5e1b99b2493
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h bc0792db4ff887e4884d386188527e1cd7c16d496c8f3ba23333e68cee3e4f78
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 59560d079fab4b91ec50855cc60349da178209c38fb6dae674ff874ccfc7f5fa
-R f928c1195d839713ee78e6b46c1884f1
+P d2f9230c5c7ad6166e7d2b649f77960fa58b1cb583e529a43882753ab348413c
+R 9289556db20d60c2fec8166d36acfa2c
 U drh
-Z fae59aff59c16e3b9c2ad1cd0f0e8fa3
+Z 14ebde2517e2be2d793887e9fa413aa5
diff --git a/manifest.uuid b/manifest.uuid
index c161e3d95e..64bb1d3ded 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d2f9230c5c7ad6166e7d2b649f77960fa58b1cb583e529a43882753ab348413c
\ No newline at end of file
+7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 96d133b41c..d794e73eac 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3150,10 +3150,10 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** ^If [URI filename] interpretation is enabled, and the filename argument
 ** begins with "file:", then the filename is interpreted as a URI. ^URI
 ** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is
-** set in the fourth argument to sqlite3_open_v2(), or if it has
+** set in the third argument to sqlite3_open_v2(), or if it has
 ** been enabled globally using the [SQLITE_CONFIG_URI] option with the
 ** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option.
-** As of SQLite version 3.7.7, URI filename interpretation is turned off
+** URI filename interpretation is turned off
 ** by default, but future releases of SQLite might enable URI filename
 ** interpretation by default.  See "[URI filenames]" for additional
 ** information.

From 9c6e07d2fa6876c20116d8d9c8e486699f992959 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 24 Aug 2017 20:54:42 +0000
Subject: [PATCH 043/270] Fix an incorrect hyperlink in a comment.

FossilOrigin-Name: 25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index a5acff37b1..e74b367e97 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fixes\sto\sdocumentation\sabout\sSQLITE_OPEN_URI.
-D 2017-08-24T15:43:26.350
+C Fix\san\sincorrect\shyperlink\sin\sa\scomment.
+D 2017-08-24T20:54:42.754
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -535,7 +535,7 @@ F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
+F src/wal.c 4c7c13a3c0d92b0bed157ac234679e4874afcb1990af33510542a0f35f9264b8
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701
@@ -1650,7 +1650,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d2f9230c5c7ad6166e7d2b649f77960fa58b1cb583e529a43882753ab348413c
-R 9289556db20d60c2fec8166d36acfa2c
+P 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
+R 0a9008b1e381e229108d828896139715
 U drh
-Z 14ebde2517e2be2d793887e9fa413aa5
+Z 0f9a691a1b4c4aa8a5eacf3d9e1dd571
diff --git a/manifest.uuid b/manifest.uuid
index 64bb1d3ded..404de58199 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
\ No newline at end of file
+25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 09f605fe57..45f1bc8c84 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -3119,7 +3119,7 @@ int sqlite3WalFrames(
     ** an out-of-order write following a WAL restart could result in
     ** database corruption.  See the ticket:
     **
-    **     http://localhost:591/sqlite/info/ff5be73dee
+    **     https://sqlite.org/src/info/ff5be73dee
     */
     if( pWal->syncHeader && sync_flags ){
       rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK);

From daaae7b9d124961d4c6bae0cfd128aab76671404 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 25 Aug 2017 01:14:43 +0000
Subject: [PATCH 044/270] Update the mechanism used to keep track of what kind
 of syncing to do for WAL transaction commits and checkpoint operations.  Use
 the checkpoint-style of syncing to sync the header of a new or restarted WAL
 file.

FossilOrigin-Name: bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168
---
 manifest       | 23 +++++++++++++----------
 manifest.uuid  |  2 +-
 src/os.c       |  2 +-
 src/pager.c    | 38 ++++++++++++++++++++++----------------
 src/wal.c      | 20 +++++++++-----------
 src/wal.h      |  8 ++++----
 test/wal2.test |  6 +++---
 7 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/manifest b/manifest
index e74b367e97..cfd5eb0e64 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sincorrect\shyperlink\sin\sa\scomment.
-D 2017-08-24T20:54:42.754
+C Update\sthe\smechanism\sused\sto\skeep\strack\sof\swhat\skind\sof\ssyncing\sto\sdo\sfor\nWAL\stransaction\scommits\sand\scheckpoint\soperations.\s\sUse\sthe\scheckpoint-style\nof\ssyncing\sto\ssync\sthe\sheader\sof\sa\snew\sor\srestarted\sWAL\sfile.
+D 2017-08-25T01:14:43.498
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -436,14 +436,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
-F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
+F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c c1dc0609f04a0659519bb2b8ca1440a64b0ad82b6c2afd675f1a50f6c918321a
+F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@@ -535,8 +535,8 @@ F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 4c7c13a3c0d92b0bed157ac234679e4874afcb1990af33510542a0f35f9264b8
-F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
+F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
+F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
@@ -1486,7 +1486,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7
+F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1650,7 +1650,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
-R 0a9008b1e381e229108d828896139715
+P 25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
+R 0c45c0346ff1a3d7a9dbb07f83f4f723
+T *branch * wal-sync-refactor
+T *sym-wal-sync-refactor *
+T -sym-trunk *
 U drh
-Z 0f9a691a1b4c4aa8a5eacf3d9e1dd571
+Z 5b1634e4e5e7d2e25acb68caf19e0213
diff --git a/manifest.uuid b/manifest.uuid
index 404de58199..cfce901f7c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
\ No newline at end of file
+bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168
\ No newline at end of file
diff --git a/src/os.c b/src/os.c
index 5cf0014794..18277df356 100644
--- a/src/os.c
+++ b/src/os.c
@@ -98,7 +98,7 @@ int sqlite3OsTruncate(sqlite3_file *id, i64 size){
 }
 int sqlite3OsSync(sqlite3_file *id, int flags){
   DO_OS_MALLOC_TEST(id);
-  return id->pMethods->xSync(id, flags);
+  return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK;
 }
 int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){
   DO_OS_MALLOC_TEST(id);
diff --git a/src/pager.c b/src/pager.c
index 5986afdae1..2e5977fe43 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -616,6 +616,18 @@ struct PagerSavepoint {
 **   is set to zero in all other states. In PAGER_ERROR state, Pager.errCode 
 **   is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX 
 **   sub-codes.
+**
+** syncFlags, walSyncFlags
+**
+**   syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03).
+**   syncFlags is used for rollback mode.  walSyncFlags is used for WAL mode
+**   and contains the flags used to sync the checkpoint operations in the
+**   lower two bits, and sync flags used for transaction commits in the WAL
+**   file in bits 0x04 and 0x08.  In other words, to get the correct sync flags
+**   for checkpoint operations, use (walSyncFlags&0x03) and to get the correct
+**   sync flags for transaction commit, use ((walSyncFlags>>2)&0x03).  Note
+**   that with synchronous=NORMAL in WAL mode, transaction commit is not synced
+**   meaning that the 0x04 and 0x08 bits are both zero.
 */
 struct Pager {
   sqlite3_vfs *pVfs;          /* OS functions to use for IO */
@@ -625,9 +637,8 @@ struct Pager {
   u8 noSync;                  /* Do not sync the journal if true */
   u8 fullSync;                /* Do extra syncs of the journal for robustness */
   u8 extraSync;               /* sync directory after journal delete */
-  u8 ckptSyncFlags;           /* SYNC_NORMAL or SYNC_FULL for checkpoint */
-  u8 walSyncFlags;            /* SYNC_NORMAL or SYNC_FULL for wal writes */
   u8 syncFlags;               /* SYNC_NORMAL or SYNC_FULL otherwise */
+  u8 walSyncFlags;            /* See description above */
   u8 tempFile;                /* zFilename is a temporary or immutable file */
   u8 noLock;                  /* Do not lock (except in WAL mode) */
   u8 readOnly;                /* True for a read-only database */
@@ -3598,20 +3609,17 @@ void sqlite3PagerSetFlags(
   }
   if( pPager->noSync ){
     pPager->syncFlags = 0;
-    pPager->ckptSyncFlags = 0;
   }else if( pgFlags & PAGER_FULLFSYNC ){
     pPager->syncFlags = SQLITE_SYNC_FULL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
-  }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){
-    pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
   }else{
     pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
   }
-  pPager->walSyncFlags = pPager->syncFlags;
+  pPager->walSyncFlags = (pPager->syncFlags<<2);
   if( pPager->fullSync ){
-    pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS;
+    pPager->walSyncFlags |= pPager->syncFlags;
+  }
+  if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){
+    pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2);
   }
   if( pgFlags & PAGER_CACHESPILL ){
     pPager->doNotSpill &= ~SPILLFLAG_OFF;
@@ -4110,7 +4118,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){
   pPager->exclusiveMode = 0;
 #ifndef SQLITE_OMIT_WAL
   assert( db || pPager->pWal==0 );
-  sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize,
+  sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,
       (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp)
   );
   pPager->pWal = 0;
@@ -4918,13 +4926,11 @@ act_like_temp_file:
     assert( pPager->extraSync==0 );
     assert( pPager->syncFlags==0 );
     assert( pPager->walSyncFlags==0 );
-    assert( pPager->ckptSyncFlags==0 );
   }else{
     pPager->fullSync = 1;
     pPager->extraSync = 0;
     pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS;
-    pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
+    pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2);
   }
   /* pPager->pFirst = 0; */
   /* pPager->pFirstSynced = 0; */
@@ -7372,7 +7378,7 @@ int sqlite3PagerCheckpoint(
     rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode,
         (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler),
         pPager->pBusyHandlerArg,
-        pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
+        pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
         pnLog, pnCkpt
     );
   }
@@ -7529,7 +7535,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
   if( rc==SQLITE_OK && pPager->pWal ){
     rc = pagerExclusiveLock(pPager);
     if( rc==SQLITE_OK ){
-      rc = sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags,
+      rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags,
                            pPager->pageSize, (u8*)pPager->pTmpSpace);
       pPager->pWal = 0;
       pagerFixMaplimit(pPager);
diff --git a/src/wal.c b/src/wal.c
index 45f1bc8c84..9930b84421 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1798,9 +1798,7 @@ static int walCheckpoint(
       pInfo->nBackfillAttempted = mxSafeFrame;
 
       /* Sync the WAL to disk */
-      if( sync_flags ){
-        rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
-      }
+      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
 
       /* If the database may grow as a result of this checkpoint, hint
       ** about the eventual size of the db file to the VFS layer.
@@ -1841,8 +1839,8 @@ static int walCheckpoint(
           i64 szDb = pWal->hdr.nPage*(i64)szPage;
           testcase( IS_BIG_INT(szDb) );
           rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
-          if( rc==SQLITE_OK && sync_flags ){
-            rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
+          if( rc==SQLITE_OK ){
+            rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
           }
         }
         if( rc==SQLITE_OK ){
@@ -2948,8 +2946,8 @@ static int walWriteToLog(
     iOffset += iFirstAmt;
     iAmt -= iFirstAmt;
     pContent = (void*)(iFirstAmt + (char*)pContent);
-    assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) );
-    rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK);
+    assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 );
+    rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags));
     if( iAmt==0 || rc ) return rc;
   }
   rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset);
@@ -3121,8 +3119,8 @@ int sqlite3WalFrames(
     **
     **     https://sqlite.org/src/info/ff5be73dee
     */
-    if( pWal->syncHeader && sync_flags ){
-      rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK);
+    if( pWal->syncHeader ){
+      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
       if( rc ) return rc;
     }
   }
@@ -3197,7 +3195,7 @@ int sqlite3WalFrames(
   ** sector boundary is synced; the part of the last frame that extends
   ** past the sector boundary is written after the sync.
   */
-  if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
+  if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){
     int bSync = 1;
     if( pWal->padToSectorBoundary ){
       int sectorSize = sqlite3SectorSize(pWal->pWalFd);
@@ -3213,7 +3211,7 @@ int sqlite3WalFrames(
     }
     if( bSync ){
       assert( rc==SQLITE_OK );
-      rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK);
+      rc = sqlite3OsSync(w.pFd, WAL_SYNC_FLAGS(sync_flags));
     }
   }
 
diff --git a/src/wal.h b/src/wal.h
index 4f6d01dad6..d97300a684 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -19,11 +19,11 @@
 
 #include "sqliteInt.h"
 
-/* Additional values that can be added to the sync_flags argument of
-** sqlite3WalFrames():
+/* Macros for extracting appropriate sync flags for either transaction
+** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)):
 */
-#define WAL_SYNC_TRANSACTIONS  0x20   /* Sync at the end of each transaction */
-#define SQLITE_SYNC_MASK       0x13   /* Mask off the SQLITE_SYNC_* values */
+#define WAL_SYNC_FLAGS(X)   ((X)&0x03)
+#define CKPT_SYNC_FLAGS(X)  (((X)>>2)&0x03)
 
 #ifdef SQLITE_OMIT_WAL
 # define sqlite3WalOpen(x,y,z)                   0
diff --git a/test/wal2.test b/test/wal2.test
index 0b15b15461..bc170ad3ff 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1191,7 +1191,7 @@ if {$::tcl_platform(platform) == "unix"} {
 #
 foreach {tn sql reslist} {
   1 { }                                 {10 0 4 0 6 0}
-  2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2}
+  2 { PRAGMA checkpoint_fullfsync = 1 } {10 6 4 3 6 3}
   3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0}
 } {
   ifcapable default_ckptfullfsync {
@@ -1261,8 +1261,8 @@ foreach {tn settings restart_sync commit_sync ckpt_sync} {
   6  {0 1 full}    {0 2}  {0 1}  {0 2}
 
   7  {1 0 off}     {0 0}  {0 0}  {0 0}
-  8  {1 0 normal}  {1 0}  {0 0}  {0 2}
-  9  {1 0 full}    {2 0}  {1 0}  {0 2}
+  8  {1 0 normal}  {0 1}  {0 0}  {0 2}
+  9  {1 0 full}    {1 1}  {1 0}  {0 2}
 
   10 {1 1 off}     {0 0}  {0 0}  {0 0}
   11 {1 1 normal}  {0 1}  {0 0}  {0 2}

From 1c305122a22464711680d8099db22ba6850e85c1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 25 Aug 2017 09:17:14 +0000
Subject: [PATCH 045/270] Avoid returning duplicate rows in experimental
 pragmas "pragma_list", "module_list" and "function_list".

FossilOrigin-Name: b79cc8dc88c8ae03daff1290fd650b2b0e6f673ec9d83be6a533a57172930190
---
 manifest          | 15 +++++------
 manifest.uuid     |  2 +-
 src/pragma.c      |  4 ---
 test/pragma5.test | 64 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 73 insertions(+), 12 deletions(-)
 create mode 100644 test/pragma5.test

diff --git a/manifest b/manifest
index e74b367e97..bfb51fad1c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sincorrect\shyperlink\sin\sa\scomment.
-D 2017-08-24T20:54:42.754
+C Avoid\sreturning\sduplicate\srows\sin\sexperimental\spragmas\s"pragma_list",\n"module_list"\sand\s"function_list".
+D 2017-08-25T09:17:14.786
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -449,7 +449,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
-F src/pragma.c cd6aeda3587be6c5c08f9b2d45eae6068666a03c9d077c8c43cdb85fb0aa70f2
+F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9e880c0efb5d7f9101bb34c0a87daf6e1e5284c34024fdb811e67bb02fdd299b
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -1098,6 +1098,7 @@ F test/pragma.test f274259d6393b6681eb433beb8dd39a26ec06a4431052a4880b43b84912a3
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264
+F test/pragma5.test fd517f42ee847e126afbbbd9fd0fb9e5a4a61a962496a350adb8a22583fbdc37
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
 F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad
@@ -1650,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
-R 0a9008b1e381e229108d828896139715
-U drh
-Z 0f9a691a1b4c4aa8a5eacf3d9e1dd571
+P 25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
+R 7b2e7d0666727914b868e680cae90457
+U dan
+Z 2b958aabca575686d03fef3a95382582
diff --git a/manifest.uuid b/manifest.uuid
index 404de58199..98964c88d8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
\ No newline at end of file
+b79cc8dc88c8ae03daff1290fd650b2b0e6f673ec9d83be6a533a57172930190
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index e640c7ebe5..2619f2944b 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1234,13 +1234,11 @@ void sqlite3Pragma(
     for(i=0; i<SQLITE_FUNC_HASH_SZ; i++){
       for(p=sqlite3BuiltinFunctions.a[i]; p; p=p->u.pHash ){
         sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 1);
-        sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2);
       }
     }
     for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){
       p = (FuncDef*)sqliteHashData(j);
       sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 0);
-      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2);
     }
   }
   break;
@@ -1252,7 +1250,6 @@ void sqlite3Pragma(
     for(j=sqliteHashFirst(&db->aModule); j; j=sqliteHashNext(j)){
       Module *pMod = (Module*)sqliteHashData(j);
       sqlite3VdbeMultiLoad(v, 1, "s", pMod->zName);
-      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
     }
   }
   break;
@@ -1262,7 +1259,6 @@ void sqlite3Pragma(
     int i;
     for(i=0; i<ArraySize(aPragmaName); i++){
       sqlite3VdbeMultiLoad(v, 1, "s", aPragmaName[i].zName);
-      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
     }
   }
   break;
diff --git a/test/pragma5.test b/test/pragma5.test
new file mode 100644
index 0000000000..625ab92012
--- /dev/null
+++ b/test/pragma5.test
@@ -0,0 +1,64 @@
+# 2017 August 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+# This file implements tests for the PRAGMA command. Specifically,
+# those pragmas enabled at build time by setting:
+#
+#   -DSQLITE_INTROSPECTION_PRAGMAS
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix pragma5
+
+if { [catch {db one "SELECT count(*) FROM pragma_function_list"}] } {
+  finish_test
+  return
+}
+
+db function external external
+
+do_execsql_test 1.0 {
+  PRAGMA table_info(pragma_function_list)
+} {
+  0 name {} 0 {} 0 
+  1 builtin {} 0 {} 0
+}
+do_execsql_test 1.1 {
+  SELECT * FROM pragma_function_list WHERE name='upper'
+} {upper 1}
+do_execsql_test 1.2 {
+  SELECT * FROM pragma_function_list WHERE name LIKE 'exter%';
+} {external 0}
+
+ifcapable fts5 {
+  do_execsql_test 2.0 {
+    PRAGMA table_info(pragma_module_list)
+  } {
+    0 name {} 0 {} 0 
+  }
+  do_execsql_test 2.1 {
+    SELECT * FROM pragma_module_list WHERE name='fts5'
+  } {fts5}
+}
+
+do_execsql_test 3.0 {
+  PRAGMA table_info(pragma_pragma_list)
+} {
+  0 name {} 0 {} 0 
+}
+do_execsql_test 3.1 {
+  SELECT * FROM pragma_pragma_list WHERE name='pragma_list'
+} {pragma_list}
+
+
+finish_test

From 6034846eb6d5123584f6a8f059885f39eaeca707 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 25 Aug 2017 13:02:48 +0000
Subject: [PATCH 046/270] Convert a branch made unreachable by [59560d07] into
 an assert().

FossilOrigin-Name: 2738b8db3caa6ce48d27cb5749d27b79241e6f6682b694886f6ef663e5443583
---
 manifest      | 13 ++++++-------
 manifest.uuid |  2 +-
 src/btree.c   |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 154d622e08..5d2eb2a992 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sinternal\smechanism\sused\sto\skeep\strack\sof\swhat\skind\sof\ssyncing\nto\sdo\sfor\sWAL\stransaction\scommits\sand\scheckpoint\soperations.\s\nUse\sthe\scheckpoint-style\sof\ssyncing\sto\ssync\sthe\sheader\sof\sa\snew\nor\srestarted\sWAL\sfile.
-D 2017-08-25T11:44:51.174
+C Convert\sa\sbranch\smade\sunreachable\sby\s[59560d07]\sinto\san\sassert().
+D 2017-08-25T13:02:48.882
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c f93c05dbd60551b322dbcaaf4961e87ea20eb67667ebc3181f15d2b5cb66bbaa
+F src/btree.c 83933a8d1ec8ebdcd49c6b1c6541e8d32a55ab5f7155b0e90babe26825223abd
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
@@ -1651,8 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b79cc8dc88c8ae03daff1290fd650b2b0e6f673ec9d83be6a533a57172930190 bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168
-R 537f505d2d307d3df7ac5ca0ae8a4ab7
-T +closed bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168
+P e8d23afe8478e42779ec1dd602ca2d77d4f3c26c4d54f52112c5aaa928536538
+R c47094d13357cf3081b589cea406d444
 U drh
-Z f4c9090d66a1befc7247247744f54bd9
+Z 0c1347313e1f31701f169aeedc1bd394
diff --git a/manifest.uuid b/manifest.uuid
index f317d7cf98..27c4e54395 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e8d23afe8478e42779ec1dd602ca2d77d4f3c26c4d54f52112c5aaa928536538
\ No newline at end of file
+2738b8db3caa6ce48d27cb5749d27b79241e6f6682b694886f6ef663e5443583
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index a4bcaf6c49..9b67ab2887 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -1399,7 +1399,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
         int sz = get2byte(&data[iFree+2]);
         int top = get2byte(&data[hdr+5]);
         if( iFree2 ){
-          if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+          assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */
           sz2 = get2byte(&data[iFree2+2]);
           assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize );
           memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));

From 83e8ca54d1f09db06ecc30687a16cbdcf7b29250 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 25 Aug 2017 13:34:18 +0000
Subject: [PATCH 047/270] Small performance optimization in
 sqlite3WhereBegin().

FossilOrigin-Name: 39f708d1e286931365a8992dbe7f900108ff1dad146032a284ad1dec09b11e06
---
 manifest      | 12 +++++------
 manifest.uuid |  2 +-
 src/where.c   | 59 ++++++++++++++++++++++++++-------------------------
 3 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/manifest b/manifest
index 5d2eb2a992..e7c1f7c289 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Convert\sa\sbranch\smade\sunreachable\sby\s[59560d07]\sinto\san\sassert().
-D 2017-08-25T13:02:48.882
+C Small\sperformance\soptimization\sin\ssqlite3WhereBegin().
+D 2017-08-25T13:34:18.937
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -538,7 +538,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701
+F src/where.c 0aaa1b085a018c1c2e2da367b48491ce2686aea351bd17772c46b7e04eb51e3d
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
 F src/wherecode.c e7be3b7f4c11908500cdf02b299d190d3742659533f58e0f4047962fdb5a48da
 F src/whereexpr.c fe1fe600d7334e91f3d9d487021362d543fba8ab2f1be5e0d68063d619379c05
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e8d23afe8478e42779ec1dd602ca2d77d4f3c26c4d54f52112c5aaa928536538
-R c47094d13357cf3081b589cea406d444
+P 2738b8db3caa6ce48d27cb5749d27b79241e6f6682b694886f6ef663e5443583
+R be107142fb32b5eeca2b38bf38398332
 U drh
-Z 0c1347313e1f31701f169aeedc1bd394
+Z ca59cf9901a72073d38774a5b3f91248
diff --git a/manifest.uuid b/manifest.uuid
index 27c4e54395..1cc2968043 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2738b8db3caa6ce48d27cb5749d27b79241e6f6682b694886f6ef663e5443583
\ No newline at end of file
+39f708d1e286931365a8992dbe7f900108ff1dad146032a284ad1dec09b11e06
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 5402977c30..d414b936d5 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4537,37 +4537,38 @@ WhereInfo *sqlite3WhereBegin(
     if( wctrlFlags & WHERE_WANT_DISTINCT ){
       pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
     }
-  }
-
-  /* Assign a bit from the bitmask to every term in the FROM clause.
-  **
-  ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
-  **
-  ** The rule of the previous sentence ensures thta if X is the bitmask for
-  ** a table T, then X-1 is the bitmask for all other tables to the left of T.
-  ** Knowing the bitmask for all tables to the left of a left join is
-  ** important.  Ticket #3015.
-  **
-  ** Note that bitmasks are created for all pTabList->nSrc tables in
-  ** pTabList, not just the first nTabList tables.  nTabList is normally
-  ** equal to pTabList->nSrc but might be shortened to 1 if the
-  ** WHERE_OR_SUBCLAUSE flag is set.
-  */
-  for(ii=0; ii<pTabList->nSrc; ii++){
-    createMask(pMaskSet, pTabList->a[ii].iCursor);
-    sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
-  }
-#ifdef SQLITE_DEBUG
-  {
-    Bitmask mx = 0;
-    for(ii=0; ii<pTabList->nSrc; ii++){
-      Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
-      assert( m>=mx );
-      mx = m;
+  }else{
+    /* Assign a bit from the bitmask to every term in the FROM clause.
+    **
+    ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
+    **
+    ** The rule of the previous sentence ensures thta if X is the bitmask for
+    ** a table T, then X-1 is the bitmask for all other tables to the left of T.
+    ** Knowing the bitmask for all tables to the left of a left join is
+    ** important.  Ticket #3015.
+    **
+    ** Note that bitmasks are created for all pTabList->nSrc tables in
+    ** pTabList, not just the first nTabList tables.  nTabList is normally
+    ** equal to pTabList->nSrc but might be shortened to 1 if the
+    ** WHERE_OR_SUBCLAUSE flag is set.
+    */
+    ii = 0;
+    do{
+      createMask(pMaskSet, pTabList->a[ii].iCursor);
+      sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
+    }while( (++ii)<pTabList->nSrc );
+  #ifdef SQLITE_DEBUG
+    {
+      Bitmask mx = 0;
+      for(ii=0; ii<pTabList->nSrc; ii++){
+        Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
+        assert( m>=mx );
+        mx = m;
+      }
     }
+  #endif
   }
-#endif
-
+  
   /* Analyze all of the subexpressions. */
   sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
   if( db->mallocFailed ) goto whereBeginError;

From 36494b8bc681cc27b145674da9bb380eeac7a371 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 25 Aug 2017 15:43:34 +0000
Subject: [PATCH 048/270] Size and performance optimization for sqlite3Init().

FossilOrigin-Name: 776d91284a891bec64d6f649ff17de898b6ac6f66dd3d2afccc394a012a40c7d
---
 manifest      | 12 +++++------
 manifest.uuid |  2 +-
 src/prepare.c | 59 +++++++++++++++++++++++----------------------------
 3 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/manifest b/manifest
index e7c1f7c289..c65dc5b85f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\sperformance\soptimization\sin\ssqlite3WhereBegin().
-D 2017-08-25T13:34:18.937
+C Size\sand\sperformance\soptimization\sfor\ssqlite3Init().
+D 2017-08-25T15:43:34.075
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -451,7 +451,7 @@ F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
 F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
 F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c 9e880c0efb5d7f9101bb34c0a87daf6e1e5284c34024fdb811e67bb02fdd299b
+F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2738b8db3caa6ce48d27cb5749d27b79241e6f6682b694886f6ef663e5443583
-R be107142fb32b5eeca2b38bf38398332
+P 39f708d1e286931365a8992dbe7f900108ff1dad146032a284ad1dec09b11e06
+R fe3e8a19bfb43e621f9a86b9a7564beb
 U drh
-Z ca59cf9901a72073d38774a5b3f91248
+Z 5ffd42d30ddfce1213880dd5c5ac3e27
diff --git a/manifest.uuid b/manifest.uuid
index 1cc2968043..bd597aa8ea 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-39f708d1e286931365a8992dbe7f900108ff1dad146032a284ad1dec09b11e06
\ No newline at end of file
+776d91284a891bec64d6f649ff17de898b6ac6f66dd3d2afccc394a012a40c7d
\ No newline at end of file
diff --git a/src/prepare.c b/src/prepare.c
index 9f0704c947..8ff27fec6c 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -150,6 +150,8 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){
   assert( sqlite3_mutex_held(db->mutex) );
   assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
 
+  db->init.busy = 1;
+
   /* Construct the in-memory representation schema tables (sqlite_master or
   ** sqlite_temp_master) by invoking the parser directly.  The appropriate
   ** table name will be inserted automatically by the parser so we can just
@@ -158,7 +160,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){
   azArg[0] = zMasterName = SCHEMA_TABLE(iDb);
   azArg[1] = "1";
   azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text,"
-                            "rootpage integer,sql text)";
+                            "rootpage int,sql text)";
   azArg[3] = 0;
   initData.db = db;
   initData.iDb = iDb;
@@ -174,10 +176,10 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){
   */
   pDb = &db->aDb[iDb];
   if( pDb->pBt==0 ){
-    if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){
-      DbSetProperty(db, 1, DB_SchemaLoaded);
-    }
-    return SQLITE_OK;
+    assert( iDb==1 );
+    DbSetProperty(db, 1, DB_SchemaLoaded);
+    rc = SQLITE_OK;
+    goto error_out;
   }
 
   /* If there is not already a read-only (or read-write) transaction opened
@@ -336,9 +338,13 @@ initone_error_out:
   sqlite3BtreeLeave(pDb->pBt);
 
 error_out:
-  if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
-    sqlite3OomFault(db);
+  if( rc ){
+    if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
+      sqlite3OomFault(db);
+    }
+    sqlite3ResetOneSchema(db, iDb);
   }
+  db->init.busy = 0;
   return rc;
 }
 
@@ -359,37 +365,24 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   assert( sqlite3_mutex_held(db->mutex) );
   assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
   assert( db->init.busy==0 );
-  rc = SQLITE_OK;
-  db->init.busy = 1;
   ENC(db) = SCHEMA_ENC(db);
-  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
-    if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue;
-    rc = sqlite3InitOne(db, i, pzErrMsg);
-    if( rc ){
-      sqlite3ResetOneSchema(db, i);
+  assert( db->nDb>0 );
+  /* Do the main schema first */
+  if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){
+    rc = sqlite3InitOne(db, 0, pzErrMsg);
+    if( rc ) return rc;
+  }
+  /* All other schemas after the main schema. The "temp" schema must be last */
+  for(i=db->nDb-1; i>0; i--){
+    if( !DbHasProperty(db, i, DB_SchemaLoaded) ){
+      rc = sqlite3InitOne(db, i, pzErrMsg);
+      if( rc ) return rc;
     }
   }
-
-  /* Once all the other databases have been initialized, load the schema
-  ** for the TEMP database. This is loaded last, as the TEMP database
-  ** schema may contain references to objects in other databases.
-  */
-#ifndef SQLITE_OMIT_TEMPDB
-  assert( db->nDb>1 );
-  if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){
-    rc = sqlite3InitOne(db, 1, pzErrMsg);
-    if( rc ){
-      sqlite3ResetOneSchema(db, 1);
-    }
-  }
-#endif
-
-  db->init.busy = 0;
-  if( rc==SQLITE_OK && commit_internal ){
+  if( commit_internal ){
     sqlite3CommitInternalChanges(db);
   }
-
-  return rc; 
+  return SQLITE_OK;
 }
 
 /*

From f49759bf030088bbe2cf35c5c642b6f265bb56f4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 25 Aug 2017 19:51:51 +0000
Subject: [PATCH 049/270] Fix a few over-length source code lines.  No
 functional changes.

FossilOrigin-Name: 1783c54ce9dcb4b2c4f2a66f6d3315e646b71b54a8899fc32a7bf0f935a21f7f
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/func.c      |  3 ++-
 src/vdbetrace.c |  2 +-
 src/wherecode.c |  6 +++---
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index c65dc5b85f..3fce59b2ae 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\sand\sperformance\soptimization\sfor\ssqlite3Init().
-D 2017-08-25T15:43:34.075
+C Fix\sa\sfew\sover-length\ssource\scode\slines.\s\sNo\sfunctional\schanges.
+D 2017-08-25T19:51:51.034
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -412,7 +412,7 @@ F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 4ca86dc65f5ea478c665a5b4fe79d05f00432c9bd82237a896b45bd376bf1217
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
-F src/func.c ed8888ae80b39f5a5d403954e4a05e0a38303523dff8143161439c142d31dec1
+F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
 F src/global.c 8a6ab6b4d91effb96ffa81b39f0d70c862abca157f8aaa194600a4a8b7923344
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -532,7 +532,7 @@ F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
 F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e3
-F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
+F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
@@ -540,7 +540,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c 0aaa1b085a018c1c2e2da367b48491ce2686aea351bd17772c46b7e04eb51e3d
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
-F src/wherecode.c e7be3b7f4c11908500cdf02b299d190d3742659533f58e0f4047962fdb5a48da
+F src/wherecode.c d246d19f5453d3f154ed8fcea892ce6d70ae4a5ddaebae34bd49d73f4c913bc7
 F src/whereexpr.c fe1fe600d7334e91f3d9d487021362d543fba8ab2f1be5e0d68063d619379c05
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39f708d1e286931365a8992dbe7f900108ff1dad146032a284ad1dec09b11e06
-R fe3e8a19bfb43e621f9a86b9a7564beb
+P 776d91284a891bec64d6f649ff17de898b6ac6f66dd3d2afccc394a012a40c7d
+R 41344988e24701db849a6d76419462a5
 U drh
-Z 5ffd42d30ddfce1213880dd5c5ac3e27
+Z b62f697e1468d19c3e527d2e161c50d8
diff --git a/manifest.uuid b/manifest.uuid
index bd597aa8ea..97e89e6b6d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-776d91284a891bec64d6f649ff17de898b6ac6f66dd3d2afccc394a012a40c7d
\ No newline at end of file
+1783c54ce9dcb4b2c4f2a66f6d3315e646b71b54a8899fc32a7bf0f935a21f7f
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 2bb9a91de6..b46578a407 100644
--- a/src/func.c
+++ b/src/func.c
@@ -865,7 +865,8 @@ static void likeFunc(
 #ifdef SQLITE_TEST
     sqlite3_like_count++;
 #endif
-    sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH);
+    sqlite3_result_int(context,
+                      patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH);
   }
 }
 
diff --git a/src/vdbetrace.c b/src/vdbetrace.c
index 7b47363991..c993d9a3dd 100644
--- a/src/vdbetrace.c
+++ b/src/vdbetrace.c
@@ -82,7 +82,7 @@ char *sqlite3VdbeExpandSql(
   Mem *pVar;               /* Value of a host parameter */
   StrAccum out;            /* Accumulate the output here */
 #ifndef SQLITE_OMIT_UTF16
-  Mem utf8;                /* Used to convert UTF16 parameters into UTF8 for display */
+  Mem utf8;                /* Used to convert UTF16 into UTF8 for display */
 #endif
   char zBase[100];         /* Initial working space */
 
diff --git a/src/wherecode.c b/src/wherecode.c
index 528aeec2b0..30ff3f25c4 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1070,9 +1070,9 @@ static int whereIndexExprTransNode(Walker *p, Expr *pExpr){
 }
 
 /*
-** For an indexes on expression X, locate every instance of expression X in pExpr
-** and change that subexpression into a reference to the appropriate column of
-** the index.
+** For an indexes on expression X, locate every instance of expression X
+** in pExpr and change that subexpression into a reference to the appropriate
+** column of the index.
 */
 static void whereIndexExprTrans(
   Index *pIdx,      /* The Index */

From 20554381d9d68a40500ddea1e745573ecebee5ee Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 28 Aug 2017 11:12:57 +0000
Subject: [PATCH 050/270] Fix the ".dump", ".schema", and ".fullschema"
 commands of the command-line shell so that they work even if PRAGMA
 empty_result_callbacks is enabled. Fix for ticket [02f0f4c54f281].

FossilOrigin-Name: cf0d3715caac9149e65bb4802fd179d0952cfaf9add17ac243c6ca87cbd6e6b7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/shell.c   |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 3fce59b2ae..a31b3715e7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sfew\sover-length\ssource\scode\slines.\s\sNo\sfunctional\schanges.
-D 2017-08-25T19:51:51.034
+C Fix\sthe\s".dump",\s".schema",\sand\s".fullschema"\scommands\sof\sthe\scommand-line\nshell\sso\sthat\sthey\swork\seven\sif\sPRAGMA\sempty_result_callbacks\sis\senabled.\nFix\sfor\sticket\s[02f0f4c54f281].
+D 2017-08-28T11:12:57.851
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -457,7 +457,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
-F src/shell.c 8f2a8b9e4ffe4f4596b1690dd628cd355d5605257e14ddba83daf5422e0e39af
+F src/shell.c 117305aab365a0448505d8cfcc27d58b0182ea314f0201bd26c340a5717419a4
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
 F src/sqlite.h.in a8e60396a73996a12a153299943f45fe59202c89bb1a46bab203a5e1b99b2493
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 776d91284a891bec64d6f649ff17de898b6ac6f66dd3d2afccc394a012a40c7d
-R 41344988e24701db849a6d76419462a5
+P 1783c54ce9dcb4b2c4f2a66f6d3315e646b71b54a8899fc32a7bf0f935a21f7f
+R 5eea66808a45f5f0657c113630b948a8
 U drh
-Z b62f697e1468d19c3e527d2e161c50d8
+Z d811ef40d0e65caf3cded0793706c012
diff --git a/manifest.uuid b/manifest.uuid
index 97e89e6b6d..78c2eaf7f4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1783c54ce9dcb4b2c4f2a66f6d3315e646b71b54a8899fc32a7bf0f935a21f7f
\ No newline at end of file
+cf0d3715caac9149e65bb4802fd179d0952cfaf9add17ac243c6ca87cbd6e6b7
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index bd7853f65d..b78da38701 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -2659,6 +2659,7 @@ static int shell_callback(
   int i;
   ShellState *p = (ShellState*)pArg;
 
+  if( azArg==0 ) return 0;
   switch( p->cMode ){
     case MODE_Line: {
       int w = 5;
@@ -3009,6 +3010,7 @@ static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){
   ShellText *p = (ShellText*)pArg;
   int i;
   UNUSED_PARAMETER(az);
+  if( azArg==0 ) return 0;
   if( p->n ) appendText(p, "|", 0);
   for(i=0; i<nArg; i++){
     if( i ) appendText(p, ",", 0);
@@ -3890,7 +3892,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   ShellState *p = (ShellState *)pArg;
 
   UNUSED_PARAMETER(azNotUsed);
-  if( nArg!=3 ) return 1;
+  if( nArg!=3 || azArg==0 ) return 0;
   zTable = azArg[0];
   zType = azArg[1];
   zSql = azArg[2];

From b3c4523c586f370bc3c0080d0dae3d05d94e674d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 28 Aug 2017 14:33:27 +0000
Subject: [PATCH 051/270] Backport changes to shell.c into shell.c.in.

FossilOrigin-Name: 2348045fbc3fc99e2b46095cc86db99815cd1f9254d30a3b72c2b15c02076a84
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 src/shell.c.in    |  6 ++++--
 tool/mkshellc.tcl |  3 ++-
 4 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index a31b3715e7..c9965634fa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s".dump",\s".schema",\sand\s".fullschema"\scommands\sof\sthe\scommand-line\nshell\sso\sthat\sthey\swork\seven\sif\sPRAGMA\sempty_result_callbacks\sis\senabled.\nFix\sfor\sticket\s[02f0f4c54f281].
-D 2017-08-28T11:12:57.851
+C Backport\schanges\sto\sshell.c\sinto\sshell.c.in.
+D 2017-08-28T14:33:27.858
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -458,7 +458,7 @@ F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c 117305aab365a0448505d8cfcc27d58b0182ea314f0201bd26c340a5717419a4
-F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
+F src/shell.c.in 31cd78c3f7a1ac5375bd47afd0585b04266e82fe33a3514d788d32d913d6ce5b
 F src/sqlite.h.in a8e60396a73996a12a153299943f45fe59202c89bb1a46bab203a5e1b99b2493
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1588,7 +1588,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 69c38ecd7b74b2b0799a35ce20e1e3998e504d8c99c100ca4b98ae9d8f6279bc
+F tool/mkshellc.tcl 8743a62e12ab67741f63f3e8ea00c482f8fa50ae3d3bca16b38754641777bf13
 F tool/mksourceid.c 30966d568654a4fd962fb324753e49429b7379e1f72d2be489ade963121f5943
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1783c54ce9dcb4b2c4f2a66f6d3315e646b71b54a8899fc32a7bf0f935a21f7f
-R 5eea66808a45f5f0657c113630b948a8
+P cf0d3715caac9149e65bb4802fd179d0952cfaf9add17ac243c6ca87cbd6e6b7
+R a662f367e40c894310c1263328998be4
 U drh
-Z d811ef40d0e65caf3cded0793706c012
+Z f8feda6acc9937dc005c3c1a1cb14725
diff --git a/manifest.uuid b/manifest.uuid
index 78c2eaf7f4..4261b01f79 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf0d3715caac9149e65bb4802fd179d0952cfaf9add17ac243c6ca87cbd6e6b7
\ No newline at end of file
+2348045fbc3fc99e2b46095cc86db99815cd1f9254d30a3b72c2b15c02076a84
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 4dac4a62d5..8686159932 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1299,6 +1299,7 @@ static int shell_callback(
   int i;
   ShellState *p = (ShellState*)pArg;
 
+  if( azArg==0 ) return 0;
   switch( p->cMode ){
     case MODE_Line: {
       int w = 5;
@@ -1649,6 +1650,7 @@ static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){
   ShellText *p = (ShellText*)pArg;
   int i;
   UNUSED_PARAMETER(az);
+  if( azArg==0 ) return 0;
   if( p->n ) appendText(p, "|", 0);
   for(i=0; i<nArg; i++){
     if( i ) appendText(p, ",", 0);
@@ -2530,7 +2532,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   ShellState *p = (ShellState *)pArg;
 
   UNUSED_PARAMETER(azNotUsed);
-  if( nArg!=3 ) return 1;
+  if( nArg!=3 || azArg==0 ) return 0;
   zTable = azArg[0];
   zType = azArg[1];
   zSql = azArg[2];
@@ -6663,7 +6665,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   stdout_is_console = isatty(1);
 
 #if USE_SYSTEM_SQLITE+0!=1
-  if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){
+  if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
     utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
             sqlite3_sourceid(), SQLITE_SOURCE_ID);
     exit(1);
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index ce75234ce9..aef127e581 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -25,8 +25,9 @@ puts $out {/* DO NOT EDIT!
 ** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script.
 */}
 set in [open $topdir/src/shell.c.in rb]
-while {![eof $in]} {
+while {1} {
   set lx [gets $in]
+  if {[eof $in]} break;
   if {[regexp {^INCLUDE } $lx]} {
     set cfile [lindex $lx 1]
     puts $out "/************************* Begin $cfile ******************/"

From b2a0f75c0606a3fcac26bf08c73c10eba4b515af Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 28 Aug 2017 15:51:35 +0000
Subject: [PATCH 052/270] Remove the rarely-used scratch memory allocator. 
 This makes the code smaller, faster, and easier to maintain.  In place of the
 scratch allocator, add the SQLITE_CONFIG_SMALL_MALLOC configuration option
 that provides a hint to SQLite that large memory allocations should be
 avoided.

FossilOrigin-Name: 54b000246cfb5c7b8adb61a17357ef5a49adddde9e48e8937834d5ba0beb8a6b
---
 manifest               |  44 ++++++-------
 manifest.uuid          |   2 +-
 src/btree.c            |   4 +-
 src/global.c           |   4 +-
 src/main.c             |  26 +-------
 src/malloc.c           | 141 +----------------------------------------
 src/shell.c.in         |  38 ++---------
 src/sqlite.h.in        |  72 +++++++--------------
 src/sqliteInt.h        |   9 +--
 src/status.c           |   1 -
 src/test_malloc.c      |  41 ------------
 src/vdbesort.c         |   8 +--
 test/kvtest.c          |  14 ----
 test/lookaside.test    |   1 -
 test/memsubsys1.test   | 139 +---------------------------------------
 test/permutations.test |  12 ++--
 test/speedtest1.c      |  20 ------
 test/tester.tcl        |   8 ---
 test/wordcount.c       |   4 --
 19 files changed, 71 insertions(+), 517 deletions(-)

diff --git a/manifest b/manifest
index c9965634fa..2131ba9ff1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Backport\schanges\sto\sshell.c\sinto\sshell.c.in.
-D 2017-08-28T14:33:27.858
+C Remove\sthe\srarely-used\sscratch\smemory\sallocator.\s\sThis\smakes\sthe\scode\ssmaller,\nfaster,\sand\seasier\sto\smaintain.\s\sIn\splace\sof\sthe\sscratch\sallocator,\sadd\sthe\nSQLITE_CONFIG_SMALL_MALLOC\sconfiguration\soption\sthat\sprovides\sa\shint\sto\sSQLite\nthat\slarge\smemory\sallocations\sshould\sbe\savoided.
+D 2017-08-28T15:51:35.625
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 83933a8d1ec8ebdcd49c6b1c6541e8d32a55ab5f7155b0e90babe26825223abd
+F src/btree.c 1033b88fe50aba7d364b5a19666a9a274caa8d4c25ab7f3914221997b46af44a
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
@@ -413,7 +413,7 @@ F src/expr.c 4ca86dc65f5ea478c665a5b4fe79d05f00432c9bd82237a896b45bd376bf1217
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
-F src/global.c 8a6ab6b4d91effb96ffa81b39f0d70c862abca157f8aaa194600a4a8b7923344
+F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
@@ -421,8 +421,8 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 42f6a2660c7a1d643cc7e863d2dcd630c6aa1e8343f5478b0592120ab84c97ba
-F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
+F src/main.c 227a83d3f840d55e40360a1a8370c120f1466ea7d73b1fffb74b8f59ad0f4046
+F src/malloc.c e069cec00407e029d71fbc9440ebbb5833a629416324b592ade8fed93b045c83
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -458,13 +458,13 @@ F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c 117305aab365a0448505d8cfcc27d58b0182ea314f0201bd26c340a5717419a4
-F src/shell.c.in 31cd78c3f7a1ac5375bd47afd0585b04266e82fe33a3514d788d32d913d6ce5b
-F src/sqlite.h.in a8e60396a73996a12a153299943f45fe59202c89bb1a46bab203a5e1b99b2493
+F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469c
+F src/sqlite.h.in f18eef5b101d5f33f98ca43decb1f025c1b629f091ad77fe2190128e93938a5a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h bc0792db4ff887e4884d386188527e1cd7c16d496c8f3ba23333e68cee3e4f78
+F src/sqliteInt.h 9a283ecf57bb81e0040d243d56e91beae76f6d5762b3ac33c7f3ec6076a71d99
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
-F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
+F src/status.c 90450a491f3f59f6978ca9832023c31238f592064e405eeda971f3c0829564eb
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828
 F src/test1.c 8513b17ca4a7a9ba28748535d178b6e472ec7394ae0eea53907f2d3bcdbab2df
@@ -494,7 +494,7 @@ F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64
 F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c
 F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3
 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd
-F src/test_malloc.c c05f6c40bd6c8bfe5f1718212f81fd5687f91766
+F src/test_malloc.c 4f06a805de86be5216a127b3777ca2d5a1ff99d1a9238374ce136a47411be36c
 F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e
 F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635
 F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a
@@ -531,7 +531,7 @@ F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
-F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e3
+F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
@@ -981,7 +981,7 @@ F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb2848
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
-F test/kvtest.c d2b8cfc91047ebf6cac4f3a04f19c3a864e4ecfd683bbb65c395df450b8dc79c
+F test/kvtest.c fcb38ffe3db028a3138b4818fc098359c80dc51a0d1278a91c99c554cc1abb92
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
 F test/like.test 67d7431c9b664254febce9e90fd2f47c7c75c8b38444e2a50ef9ec2776b84ca8
@@ -999,7 +999,7 @@ F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
 F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
-F test/lookaside.test 90052e87282de256d613fcf8c9cbb845e4001d2f
+F test/lookaside.test b17c99ae3aef96a8c9fa6f6be33cc75b93d657cb791d3827302b6835b71941f7
 F test/main.test bb75e406c9b64931f3dc7e7f04626633365bb22f
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
@@ -1029,7 +1029,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e
 F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f
 F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
-F test/memsubsys1.test 6d268d0ae90f8d61a2356a1838665654d83de518
+F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e
 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08
 F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
@@ -1093,7 +1093,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 3b94f8fd431d39fac4952eb5dc38e1bb2b4518e1ac967d66f5abc815c104aeb6
+F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531
 F test/pragma.test f274259d6393b6681eb433beb8dd39a26ec06a4431052a4880b43b84912a3f58
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1219,7 +1219,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 7b1ab42b097b484c18d99e1d1c71a6a0c9c87a7a
+F test/speedtest1.c 7a6ec22e87f78ef98d523667593f5d818f6be4a1bcf5fe70d933aece058f32df
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
@@ -1257,7 +1257,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl eb7ec55fe074a909423c1de701f7c545417b8aa96787b8c3e7a79203f2cebec8
+F test/tester.tcl 9948bd856ce8a1c127f2f7900365387a42a917ce0dc87185bdd128fa5b11aff2
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1553,7 +1553,7 @@ F test/without_rowid3.test 2724c787a51a5dce09d078453a758117b4b728f1
 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a
 F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a
 F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e
-F test/wordcount.c 06efb84b7c48a4973c2c24ea06c93d00bce24389
+F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf8ac
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cf0d3715caac9149e65bb4802fd179d0952cfaf9add17ac243c6ca87cbd6e6b7
-R a662f367e40c894310c1263328998be4
+P 2348045fbc3fc99e2b46095cc86db99815cd1f9254d30a3b72c2b15c02076a84
+R dbf94ead9bb02368db4853c4bef6223c
 U drh
-Z f8feda6acc9937dc005c3c1a1cb14725
+Z 970c3285dea1ce91571a8f403e94ff33
diff --git a/manifest.uuid b/manifest.uuid
index 4261b01f79..f6193d5c82 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2348045fbc3fc99e2b46095cc86db99815cd1f9254d30a3b72c2b15c02076a84
\ No newline at end of file
+54b000246cfb5c7b8adb61a17357ef5a49adddde9e48e8937834d5ba0beb8a6b
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 9b67ab2887..9ec61507cb 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -7288,7 +7288,7 @@ static int balance_nonroot(
   /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer
   ** that is more than 6 times the database page size. */
   assert( szScratch<=6*(int)pBt->pageSize );
-  b.apCell = sqlite3ScratchMalloc( szScratch ); 
+  b.apCell = sqlite3StackAllocRaw(0, szScratch );
   if( b.apCell==0 ){
     rc = SQLITE_NOMEM_BKPT;
     goto balance_cleanup;
@@ -7866,7 +7866,7 @@ static int balance_nonroot(
   ** Cleanup before returning.
   */
 balance_cleanup:
-  sqlite3ScratchFree(b.apCell);
+  sqlite3StackFree(0, b.apCell);
   for(i=0; i<nOld; i++){
     releasePage(apOld[i]);
   }
diff --git a/src/global.c b/src/global.c
index fe63bbe140..e66be81495 100644
--- a/src/global.c
+++ b/src/global.c
@@ -199,6 +199,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    SQLITE_THREADSAFE==1,      /* bFullMutex */
    SQLITE_USE_URI,            /* bOpenUri */
    SQLITE_ALLOW_COVERING_INDEX_SCAN,   /* bUseCis */
+   0,                         /* bSmallMalloc */
    0x7ffffffe,                /* mxStrlen */
    0,                         /* neverCorrupt */
    SQLITE_DEFAULT_LOOKASIDE,  /* szLookaside, nLookaside */
@@ -211,9 +212,6 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0, 0,                      /* mnHeap, mxHeap */
    SQLITE_DEFAULT_MMAP_SIZE,  /* szMmap */
    SQLITE_MAX_MMAP_SIZE,      /* mxMmap */
-   (void*)0,                  /* pScratch */
-   0,                         /* szScratch */
-   0,                         /* nScratch */
    (void*)0,                  /* pPage */
    0,                         /* szPage */
    SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */
diff --git a/src/main.c b/src/main.c
index 2d81358370..212f7d6b5f 100644
--- a/src/main.c
+++ b/src/main.c
@@ -436,14 +436,8 @@ int sqlite3_config(int op, ...){
       sqlite3GlobalConfig.bMemstat = va_arg(ap, int);
       break;
     }
-    case SQLITE_CONFIG_SCRATCH: {
-      /* EVIDENCE-OF: R-08404-60887 There are three arguments to
-      ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from
-      ** which the scratch allocations will be drawn, the size of each scratch
-      ** allocation (sz), and the maximum number of scratch allocations (N). */
-      sqlite3GlobalConfig.pScratch = va_arg(ap, void*);
-      sqlite3GlobalConfig.szScratch = va_arg(ap, int);
-      sqlite3GlobalConfig.nScratch = va_arg(ap, int);
+    case SQLITE_CONFIG_SMALL_MALLOC: {
+      sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int);
       break;
     }
     case SQLITE_CONFIG_PAGECACHE: {
@@ -3802,22 +3796,6 @@ int sqlite3_test_control(int op, ...){
     }
 #endif 
 
-    /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree);
-    **
-    ** Pass pFree into sqlite3ScratchFree(). 
-    ** If sz>0 then allocate a scratch buffer into pNew.  
-    */
-    case SQLITE_TESTCTRL_SCRATCHMALLOC: {
-      void *pFree, **ppNew;
-      int sz;
-      sz = va_arg(ap, int);
-      ppNew = va_arg(ap, void**);
-      pFree = va_arg(ap, void*);
-      if( sz ) *ppNew = sqlite3ScratchMalloc(sz);
-      sqlite3ScratchFree(pFree);
-      break;
-    }
-
     /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff);
     **
     ** If parameter onoff is non-zero, configure the wrappers so that all
diff --git a/src/malloc.c b/src/malloc.c
index 5eb65bf623..78a50b49c5 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -32,14 +32,6 @@ int sqlite3_release_memory(int n){
 #endif
 }
 
-/*
-** An instance of the following object records the location of
-** each unused scratch buffer.
-*/
-typedef struct ScratchFreeslot {
-  struct ScratchFreeslot *pNext;   /* Next unused scratch buffer */
-} ScratchFreeslot;
-
 /*
 ** State information local to the memory allocation subsystem.
 */
@@ -47,22 +39,12 @@ static SQLITE_WSD struct Mem0Global {
   sqlite3_mutex *mutex;         /* Mutex to serialize access */
   sqlite3_int64 alarmThreshold; /* The soft heap limit */
 
-  /*
-  ** Pointers to the end of sqlite3GlobalConfig.pScratch memory
-  ** (so that a range test can be used to determine if an allocation
-  ** being freed came from pScratch) and a pointer to the list of
-  ** unused scratch allocations.
-  */
-  void *pScratchEnd;
-  ScratchFreeslot *pScratchFree;
-  u32 nScratchFree;
-
   /*
   ** True if heap is nearly "full" where "full" is defined by the
   ** sqlite3_soft_heap_limit() setting.
   */
   int nearlyFull;
-} mem0 = { 0, 0, 0, 0, 0, 0 };
+} mem0 = { 0, 0, 0 };
 
 #define mem0 GLOBAL(struct Mem0Global, mem0)
 
@@ -132,28 +114,6 @@ int sqlite3MallocInit(void){
   }
   memset(&mem0, 0, sizeof(mem0));
   mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM);
-  if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100
-      && sqlite3GlobalConfig.nScratch>0 ){
-    int i, n, sz;
-    ScratchFreeslot *pSlot;
-    sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch);
-    sqlite3GlobalConfig.szScratch = sz;
-    pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch;
-    n = sqlite3GlobalConfig.nScratch;
-    mem0.pScratchFree = pSlot;
-    mem0.nScratchFree = n;
-    for(i=0; i<n-1; i++){
-      pSlot->pNext = (ScratchFreeslot*)(sz+(char*)pSlot);
-      pSlot = pSlot->pNext;
-    }
-    pSlot->pNext = 0;
-    mem0.pScratchEnd = (void*)&pSlot[1];
-  }else{
-    mem0.pScratchEnd = 0;
-    sqlite3GlobalConfig.pScratch = 0;
-    sqlite3GlobalConfig.szScratch = 0;
-    sqlite3GlobalConfig.nScratch = 0;
-  }
   if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512
       || sqlite3GlobalConfig.nPage<=0 ){
     sqlite3GlobalConfig.pPage = 0;
@@ -304,105 +264,6 @@ void *sqlite3_malloc64(sqlite3_uint64 n){
   return sqlite3Malloc(n);
 }
 
-/*
-** Each thread may only have a single outstanding allocation from
-** xScratchMalloc().  We verify this constraint in the single-threaded
-** case by setting scratchAllocOut to 1 when an allocation
-** is outstanding clearing it when the allocation is freed.
-*/
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-static int scratchAllocOut = 0;
-#endif
-
-
-/*
-** Allocate memory that is to be used and released right away.
-** This routine is similar to alloca() in that it is not intended
-** for situations where the memory might be held long-term.  This
-** routine is intended to get memory to old large transient data
-** structures that would not normally fit on the stack of an
-** embedded processor.
-*/
-void *sqlite3ScratchMalloc(int n){
-  void *p;
-  assert( n>0 );
-
-  sqlite3_mutex_enter(mem0.mutex);
-  sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n);
-  if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){
-    p = mem0.pScratchFree;
-    mem0.pScratchFree = mem0.pScratchFree->pNext;
-    mem0.nScratchFree--;
-    sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1);
-    sqlite3_mutex_leave(mem0.mutex);
-  }else{
-    sqlite3_mutex_leave(mem0.mutex);
-    p = sqlite3Malloc(n);
-    if( sqlite3GlobalConfig.bMemstat && p ){
-      sqlite3_mutex_enter(mem0.mutex);
-      sqlite3StatusUp(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p));
-      sqlite3_mutex_leave(mem0.mutex);
-    }
-    sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH);
-  }
-  assert( sqlite3_mutex_notheld(mem0.mutex) );
-
-
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-  /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch
-  ** buffers per thread.
-  **
-  ** This can only be checked in single-threaded mode.
-  */
-  assert( scratchAllocOut==0 );
-  if( p ) scratchAllocOut++;
-#endif
-
-  return p;
-}
-void sqlite3ScratchFree(void *p){
-  if( p ){
-
-#if SQLITE_THREADSAFE==0 && !defined(NDEBUG)
-    /* Verify that no more than two scratch allocation per thread
-    ** is outstanding at one time.  (This is only checked in the
-    ** single-threaded case since checking in the multi-threaded case
-    ** would be much more complicated.) */
-    assert( scratchAllocOut>=1 && scratchAllocOut<=2 );
-    scratchAllocOut--;
-#endif
-
-    if( SQLITE_WITHIN(p, sqlite3GlobalConfig.pScratch, mem0.pScratchEnd) ){
-      /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */
-      ScratchFreeslot *pSlot;
-      pSlot = (ScratchFreeslot*)p;
-      sqlite3_mutex_enter(mem0.mutex);
-      pSlot->pNext = mem0.pScratchFree;
-      mem0.pScratchFree = pSlot;
-      mem0.nScratchFree++;
-      assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch );
-      sqlite3StatusDown(SQLITE_STATUS_SCRATCH_USED, 1);
-      sqlite3_mutex_leave(mem0.mutex);
-    }else{
-      /* Release memory back to the heap */
-      assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) );
-      assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_SCRATCH) );
-      sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
-      if( sqlite3GlobalConfig.bMemstat ){
-        int iSize = sqlite3MallocSize(p);
-        sqlite3_mutex_enter(mem0.mutex);
-        sqlite3StatusDown(SQLITE_STATUS_SCRATCH_OVERFLOW, iSize);
-        sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, iSize);
-        sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1);
-        sqlite3GlobalConfig.m.xFree(p);
-        sqlite3_mutex_leave(mem0.mutex);
-      }else{
-        sqlite3GlobalConfig.m.xFree(p);
-      }
-    }
-  }
-}
-
 /*
 ** TRUE if p is a lookaside memory allocation from db
 */
diff --git a/src/shell.c.in b/src/shell.c.in
index 8686159932..bf5e335857 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -871,14 +871,13 @@ struct ShellState {
 /*
 ** These are the allowed shellFlgs values
 */
-#define SHFLG_Scratch        0x00000001 /* The --scratch option is used */
-#define SHFLG_Pagecache      0x00000002 /* The --pagecache option is used */
-#define SHFLG_Lookaside      0x00000004 /* Lookaside memory is used */
-#define SHFLG_Backslash      0x00000008 /* The --backslash option is used */
-#define SHFLG_PreserveRowid  0x00000010 /* .dump preserves rowid values */
-#define SHFLG_Newlines       0x00000020 /* .dump --newline flag */
-#define SHFLG_CountChanges   0x00000040 /* .changes setting */
-#define SHFLG_Echo           0x00000080 /* .echo or --echo setting */
+#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
+#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
+#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
+#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
+#define SHFLG_Newlines       0x00000010 /* .dump --newline flag */
+#define SHFLG_CountChanges   0x00000020 /* .changes setting */
+#define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
 
 /*
 ** Macros for testing and setting shellFlgs
@@ -1897,18 +1896,10 @@ static int display_stats(
     }
     displayStatLine(pArg, "Number of Pcache Overflow Bytes:",
        "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset);
-    if( pArg->shellFlgs & SHFLG_Scratch ){
-      displayStatLine(pArg, "Number of Scratch Allocations Used:",
-         "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset);
-    }
-    displayStatLine(pArg, "Number of Scratch Overflow Bytes:",
-       "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset);
     displayStatLine(pArg, "Largest Allocation:",
        "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset);
     displayStatLine(pArg, "Largest Pcache Allocation:",
        "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset);
-    displayStatLine(pArg, "Largest Scratch Allocation:",
-       "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset);
 #ifdef YYTRACKMAXSTACKDEPTH
     displayStatLine(pArg, "Deepest Parser Stack:",
        "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset);
@@ -5929,7 +5920,6 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "reserve",               SQLITE_TESTCTRL_RESERVE                },
       { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
       { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
-      { "scratchmalloc",         SQLITE_TESTCTRL_SCRATCHMALLOC          },
       { "byteorder",             SQLITE_TESTCTRL_BYTEORDER              },
       { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
       { "imposter",              SQLITE_TESTCTRL_IMPOSTER               },
@@ -6042,7 +6032,6 @@ static int do_meta_command(char *zLine, ShellState *p){
         case SQLITE_TESTCTRL_BITVEC_TEST:
         case SQLITE_TESTCTRL_FAULT_INSTALL:
         case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS:
-        case SQLITE_TESTCTRL_SCRATCHMALLOC:
         default:
           utf8_printf(stderr,
                       "Error: CLI support for testctrl %s not implemented\n",
@@ -6562,7 +6551,6 @@ static const char zOptions[] =
   "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
   "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
   "   -quote               set output mode to 'quote'\n"
-  "   -scratch SIZE N      use N slots of SZ bytes each for scratch memory\n"
   "   -separator SEP       set output column separator. Default: '|'\n"
   "   -stats               print memory stats before each finalize\n"
   "   -version             show SQLite version\n"
@@ -6760,16 +6748,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
       (void)cmdline_option_value(argc, argv, ++i);
 #endif
-    }else if( strcmp(z,"-scratch")==0 ){
-      int n, sz;
-      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( sz>400000 ) sz = 400000;
-      if( sz<2500 ) sz = 2500;
-      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( n>10 ) n = 10;
-      if( n<1 ) n = 1;
-      sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n);
-      data.shellFlgs |= SHFLG_Scratch;
     }else if( strcmp(z,"-pagecache")==0 ){
       int n, sz;
       sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
@@ -6913,8 +6891,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       stdin_is_interactive = 0;
     }else if( strcmp(z,"-heap")==0 ){
       i++;
-    }else if( strcmp(z,"-scratch")==0 ){
-      i+=2;
     }else if( strcmp(z,"-pagecache")==0 ){
       i+=2;
     }else if( strcmp(z,"-lookaside")==0 ){
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index d794e73eac..fd30eab9a3 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -1664,6 +1664,16 @@ struct sqlite3_mem_methods {
 ** routines with a wrapper that simulations memory allocation failure or
 ** tracks memory usage, for example. </dd>
 **
+** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt>
+** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of
+** type int, interpreted as a boolean, which if true provides a hint to
+** SQLite that it should avoid large memory allocations if possible.
+** SQLite will run faster if it is free to make large memory allocations,
+** but some application might prefer to run slower in exchange for
+** guarantees about memory fragmentation that are possible if large
+** allocations are avoided.  This hint is normally off.
+** </dd>
+**
 ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt>
 ** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int,
 ** interpreted as a boolean, which enables or disables the collection of
@@ -1681,25 +1691,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_CONFIG_SCRATCH]] <dt>SQLITE_CONFIG_SCRATCH</dt>
-** <dd> ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer
-** that SQLite can use for scratch memory.  ^(There are three arguments
-** to SQLITE_CONFIG_SCRATCH:  A pointer an 8-byte
-** aligned memory buffer from which the scratch allocations will be
-** drawn, the size of each scratch allocation (sz),
-** and the maximum number of scratch allocations (N).)^
-** The first argument must be a pointer to an 8-byte aligned buffer
-** of at least sz*N bytes of memory.
-** ^SQLite will not use more than one scratch buffers per thread.
-** ^SQLite will never request a scratch buffer that is more than 6
-** times the database page size.
-** ^If SQLite needs needs additional
-** scratch memory beyond what is provided by this configuration option, then 
-** [sqlite3_malloc()] will be used to obtain the memory needed.<p>
-** ^When the application provides any amount of scratch memory using
-** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large
-** [sqlite3_malloc|heap allocations].
-** This can help [Robson proof|prevent memory allocation failures] due to heap
-** fragmentation in low-memory embedded systems.
+** <dd> The SQLITE_CONFIG_SCRATCH option is no longer used.
 ** </dd>
 **
 ** [[SQLITE_CONFIG_PAGECACHE]] <dt>SQLITE_CONFIG_PAGECACHE</dt>
@@ -1735,8 +1727,7 @@ struct sqlite3_mem_methods {
 ** [[SQLITE_CONFIG_HEAP]] <dt>SQLITE_CONFIG_HEAP</dt>
 ** <dd> ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer 
 ** that SQLite will use for all of its dynamic memory allocation needs
-** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and
-** [SQLITE_CONFIG_PAGECACHE].
+** beyond those provided for by [SQLITE_CONFIG_PAGECACHE].
 ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled
 ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns
 ** [SQLITE_ERROR] if invoked otherwise.
@@ -1929,7 +1920,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_CONFIG_SERIALIZED    3  /* nil */
 #define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
 #define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
-#define SQLITE_CONFIG_SCRATCH       6  /* void*, int sz, int N */
+#define SQLITE_CONFIG_SCRATCH       6  /* No longer used */
 #define SQLITE_CONFIG_PAGECACHE     7  /* void*, int sz, int N */
 #define SQLITE_CONFIG_HEAP          8  /* void*, int nByte, int min */
 #define SQLITE_CONFIG_MEMSTATUS     9  /* boolean */
@@ -1950,6 +1941,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
 #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
 #define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
+#define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
 
 /*
 ** CAPI3REF: Database Connection Configuration Options
@@ -7014,7 +7006,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_RESERVE                 14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16
-#define SQLITE_TESTCTRL_SCRATCHMALLOC           17
+#define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
 #define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
 #define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
@@ -7073,8 +7065,7 @@ int sqlite3_status64(
 ** <dd>This parameter is the current amount of memory checked out
 ** using [sqlite3_malloc()], either directly or indirectly.  The
 ** figure includes calls made to [sqlite3_malloc()] by the application
-** and internal memory usage by the SQLite library.  Scratch memory
-** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache
+** and internal memory usage by the SQLite library.  Auxiliary page-cache
 ** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in
 ** this parameter.  The amount returned is the sum of the allocation
 ** sizes as reported by the xSize method in [sqlite3_mem_methods].</dd>)^
@@ -7112,29 +7103,14 @@ int sqlite3_status64(
 ** *pHighwater parameter to [sqlite3_status()] is of interest.  
 ** The value written into the *pCurrent parameter is undefined.</dd>)^
 **
-** [[SQLITE_STATUS_SCRATCH_USED]] ^(<dt>SQLITE_STATUS_SCRATCH_USED</dt>
-** <dd>This parameter returns the number of allocations used out of the
-** [scratch memory allocator] configured using
-** [SQLITE_CONFIG_SCRATCH].  The value returned is in allocations, not
-** in bytes.  Since a single thread may only have one scratch allocation
-** outstanding at time, this parameter also reports the number of threads
-** using scratch memory at the same time.</dd>)^
+** [[SQLITE_STATUS_SCRATCH_USED]] <dt>SQLITE_STATUS_SCRATCH_USED</dt>
+** <dd>No longer used.</dd>
 **
 ** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(<dt>SQLITE_STATUS_SCRATCH_OVERFLOW</dt>
-** <dd>This parameter returns the number of bytes of scratch memory
-** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH]
-** buffer and where forced to overflow to [sqlite3_malloc()].  The values
-** returned include overflows because the requested allocation was too
-** larger (that is, because the requested allocation was larger than the
-** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer
-** slots were available.
-** </dd>)^
+** <dd>No longer used.</dd>
 **
-** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(<dt>SQLITE_STATUS_SCRATCH_SIZE</dt>
-** <dd>This parameter records the largest memory allocation request
-** handed to [scratch memory allocator].  Only the value returned in the
-** *pHighwater parameter to [sqlite3_status()] is of interest.  
-** The value written into the *pCurrent parameter is undefined.</dd>)^
+** [[SQLITE_STATUS_SCRATCH_SIZE]] <dt>SQLITE_STATUS_SCRATCH_SIZE</dt>
+** <dd>No longer used.</dd>
 **
 ** [[SQLITE_STATUS_PARSER_STACK]] ^(<dt>SQLITE_STATUS_PARSER_STACK</dt>
 ** <dd>The *pHighwater parameter records the deepest parser stack. 
@@ -7147,12 +7123,12 @@ int sqlite3_status64(
 #define SQLITE_STATUS_MEMORY_USED          0
 #define SQLITE_STATUS_PAGECACHE_USED       1
 #define SQLITE_STATUS_PAGECACHE_OVERFLOW   2
-#define SQLITE_STATUS_SCRATCH_USED         3
-#define SQLITE_STATUS_SCRATCH_OVERFLOW     4
+#define SQLITE_STATUS_SCRATCH_USED         3  /* NOT USED */
+#define SQLITE_STATUS_SCRATCH_OVERFLOW     4  /* NOT USED */
 #define SQLITE_STATUS_MALLOC_SIZE          5
 #define SQLITE_STATUS_PARSER_STACK         6
 #define SQLITE_STATUS_PAGECACHE_SIZE       7
-#define SQLITE_STATUS_SCRATCH_SIZE         8
+#define SQLITE_STATUS_SCRATCH_SIZE         8  /* NOT USED */
 #define SQLITE_STATUS_MALLOC_COUNT         9
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 3689689403..55a8efe214 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3268,6 +3268,7 @@ struct Sqlite3Config {
   int bFullMutex;                   /* True to enable full mutexing */
   int bOpenUri;                     /* True to interpret filenames as URIs */
   int bUseCis;                      /* Use covering indices for full-scans */
+  int bSmallMalloc;                 /* Avoid large memory allocations if true */
   int mxStrlen;                     /* Maximum string length */
   int neverCorrupt;                 /* Database is always well-formed */
   int szLookaside;                  /* Default lookaside buffer size */
@@ -3281,9 +3282,6 @@ struct Sqlite3Config {
   int mnReq, mxReq;                 /* Min and max heap requests sizes */
   sqlite3_int64 szMmap;             /* mmap() space per open file */
   sqlite3_int64 mxMmap;             /* Maximum value for szMmap */
-  void *pScratch;                   /* Scratch memory */
-  int szScratch;                    /* Size of each scratch buffer */
-  int nScratch;                     /* Number of scratch buffers */
   void *pPage;                      /* Page cache memory */
   int szPage;                       /* Size of each page in pPage[] */
   int nPage;                        /* Number of pages in pPage[] */
@@ -3522,8 +3520,6 @@ void sqlite3DbFree(sqlite3*, void*);
 void sqlite3DbFreeNN(sqlite3*, void*);
 int sqlite3MallocSize(void*);
 int sqlite3DbMallocSize(sqlite3*, void*);
-void *sqlite3ScratchMalloc(int);
-void sqlite3ScratchFree(void*);
 void *sqlite3PageMalloc(int);
 void sqlite3PageFree(void*);
 void sqlite3MemSetDefault(void);
@@ -4385,8 +4381,7 @@ SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...);
 #endif
 #define MEMTYPE_HEAP       0x01  /* General heap allocations */
 #define MEMTYPE_LOOKASIDE  0x02  /* Heap that might have been lookaside */
-#define MEMTYPE_SCRATCH    0x04  /* Scratch allocations */
-#define MEMTYPE_PCACHE     0x08  /* Page cache allocations */
+#define MEMTYPE_PCACHE     0x04  /* Page cache allocations */
 
 /*
 ** Threading interface
diff --git a/src/status.c b/src/status.c
index 24dcad4572..b783b395d3 100644
--- a/src/status.c
+++ b/src/status.c
@@ -122,7 +122,6 @@ void sqlite3StatusHighwater(int op, int X){
                                            : sqlite3MallocMutex()) );
   assert( op==SQLITE_STATUS_MALLOC_SIZE
           || op==SQLITE_STATUS_PAGECACHE_SIZE
-          || op==SQLITE_STATUS_SCRATCH_SIZE
           || op==SQLITE_STATUS_PARSER_STACK );
   if( newValue>wsdStat.mxValue[op] ){
     wsdStat.mxValue[op] = newValue;
diff --git a/src/test_malloc.c b/src/test_malloc.c
index e8c248f958..b13d9b2b2c 100644
--- a/src/test_malloc.c
+++ b/src/test_malloc.c
@@ -887,46 +887,6 @@ static int SQLITE_TCLAPI test_memdebug_log(
   return TCL_OK;
 }
 
-/*
-** Usage:    sqlite3_config_scratch SIZE N
-**
-** Set the scratch memory buffer using SQLITE_CONFIG_SCRATCH.
-** The buffer is static and is of limited size.  N might be
-** adjusted downward as needed to accommodate the requested size.
-** The revised value of N is returned.
-**
-** A negative SIZE causes the buffer pointer to be NULL.
-*/
-static int SQLITE_TCLAPI test_config_scratch(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  int sz, N, rc;
-  Tcl_Obj *pResult;
-  static char *buf = 0;
-  if( objc!=3 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "SIZE N");
-    return TCL_ERROR;
-  }
-  if( Tcl_GetIntFromObj(interp, objv[1], &sz) ) return TCL_ERROR;
-  if( Tcl_GetIntFromObj(interp, objv[2], &N) ) return TCL_ERROR;
-  free(buf);
-  if( sz<0 ){
-    buf = 0;
-    rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, (void*)0, 0, 0);
-  }else{
-    buf = malloc( sz*N + 1 );
-    rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, buf, sz, N);
-  }
-  pResult = Tcl_NewObj();
-  Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(rc));
-  Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(N));
-  Tcl_SetObjResult(interp, pResult);
-  return TCL_OK;
-}
-
 /*
 ** Usage:    sqlite3_config_pagecache SIZE N
 **
@@ -1538,7 +1498,6 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){
      { "sqlite3_memdebug_settitle",  test_memdebug_settitle        ,0 },
      { "sqlite3_memdebug_malloc_count", test_memdebug_malloc_count ,0 },
      { "sqlite3_memdebug_log",       test_memdebug_log             ,0 },
-     { "sqlite3_config_scratch",     test_config_scratch           ,0 },
      { "sqlite3_config_pagecache",   test_config_pagecache         ,0 },
      { "sqlite3_config_alt_pcache",  test_alt_pcache               ,0 },
      { "sqlite3_status",             test_status                   ,0 },
diff --git a/src/vdbesort.c b/src/vdbesort.c
index 8ce7415737..ef5715d249 100644
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -1000,11 +1000,9 @@ int sqlite3VdbeSorterInit(
       mxCache = MIN(mxCache, SQLITE_MAX_PMASZ);
       pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache);
 
-      /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of
-      ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary
-      ** large heap allocations.
-      */
-      if( sqlite3GlobalConfig.pScratch==0 ){
+      /* Avoid large memory allocations if the application has requested
+      ** SQLITE_CONFIG_SMALL_MALLOC. */
+      if( sqlite3GlobalConfig.bSmallMalloc==0 ){
         assert( pSorter->iMemory==0 );
         pSorter->nMemory = pgsz;
         pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
diff --git a/test/kvtest.c b/test/kvtest.c
index b80adfea09..9193586a1e 100644
--- a/test/kvtest.c
+++ b/test/kvtest.c
@@ -741,16 +741,6 @@ static int display_stats(
           "Number of Pcache Overflow Bytes:     %d (max %d) bytes\n",
           iCur, iHiwtr);
   iHiwtr = iCur = -1;
-  sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset);
-  fprintf(out,
-      "Number of Scratch Allocations Used:  %d (max %d)\n",
-      iCur, iHiwtr);
-  iHiwtr = iCur = -1;
-  sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset);
-  fprintf(out,
-          "Number of Scratch Overflow Bytes:    %d (max %d) bytes\n",
-          iCur, iHiwtr);
-  iHiwtr = iCur = -1;
   sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset);
   fprintf(out, "Largest Allocation:                  %d bytes\n",
           iHiwtr);
@@ -758,10 +748,6 @@ static int display_stats(
   sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset);
   fprintf(out, "Largest Pcache Allocation:           %d bytes\n",
           iHiwtr);
-  iHiwtr = iCur = -1;
-  sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset);
-  fprintf(out, "Largest Scratch Allocation:          %d bytes\n",
-          iHiwtr);
 
   iHiwtr = iCur = -1;
   sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);
diff --git a/test/lookaside.test b/test/lookaside.test
index a89110ee1d..67a502afb2 100644
--- a/test/lookaside.test
+++ b/test/lookaside.test
@@ -33,7 +33,6 @@ test_set_config_pagecache 0 0
 
 catch {db close}
 sqlite3_shutdown
-sqlite3_config_scratch 0 0
 sqlite3_initialize
 autoinstall_test_functions
 sqlite3 db test.db
diff --git a/test/memsubsys1.test b/test/memsubsys1.test
index 36427f9bae..41bc115269 100644
--- a/test/memsubsys1.test
+++ b/test/memsubsys1.test
@@ -16,7 +16,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 sqlite3_reset_auto_extension
 
-# This test assumes that no page-cache or scratch buffers are installed
+# This test assumes that no page-cache buffers are installed
 # by default when a new database connection is opened. As a result, it
 # will not work with the "memsubsys1" permutation.
 #
@@ -156,12 +156,11 @@ do_test memsubsys1-3.2.5 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
 } 0
 
-# Test 4:  Activate both PAGECACHE and SCRATCH.
+# Test 4:  Activate PAGECACHE
 #
 db close
 sqlite3_shutdown
 sqlite3_config_pagecache [expr 1024+$xtra_size] 50
-sqlite3_config_scratch 6000 2
 sqlite3_initialize
 reset_highwater_marks
 build_test_db memsubsys1-4 {PRAGMA page_size=1024}
@@ -177,144 +176,10 @@ do_test memsubsys1-4.5 {
   set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
   expr {$maxreq<7000}
 } 1
-do_test memsubsys1-4.6 {
-  set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
-} 1
-
-# Test 5:  Activate both PAGECACHE and SCRATCH.  But make the page size is
-# such that the SCRATCH allocations are too small.
-#
-db close
-sqlite3_shutdown
-sqlite3_config_pagecache [expr 4096+$xtra_size] 24
-sqlite3_config_scratch 4000 2
-sqlite3_initialize
-reset_highwater_marks
-build_test_db memsubsys1-5 {PRAGMA page_size=4096}
-#show_memstats
-do_test memsubsys1-5.3 {
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} {/^2[34]$/}
-do_test memsubsys1-5.4 {
-  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-  expr {$maxreq>4096}
-} 1
-do_test memsubsys1-5.5 {
-  set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
-} 0
-do_test memsubsys1-5.6 {
-  set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]
-  expr {$s_ovfl>6000}
-} 1
-
-# Test 6:  Activate both PAGECACHE and SCRATCH with a 4k page size.
-# Make it so that SCRATCH is large enough
-#
-db close
-sqlite3_shutdown
-sqlite3_config_pagecache [expr 4096+$xtra_size] 24
-sqlite3_config_scratch 25300 1
-sqlite3_initialize
-reset_highwater_marks
-build_test_db memsubsys1-6 {PRAGMA page_size=4096}
-#show_memstats
-do_test memsubsys1-6.3 {
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} {/^2[34]$/}
-#do_test memsubsys1-6.4 {
-#  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-#  expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)}
-#} 1
-do_test memsubsys1-6.5 {
-  set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
-} 1
-do_test memsubsys1-6.6 {
-  set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]
-} 0
-
-# Test 7:  Activate both PAGECACHE and SCRATCH with a 4k page size.
-# Set cache_size small so that no PAGECACHE overflow occurs.  Verify
-# that maximum allocation size is small.
-#
-db close
-sqlite3_shutdown
-sqlite3_config_pagecache [expr 4096+$xtra_size] 24
-sqlite3_config_scratch 25300 1
-sqlite3_initialize
-reset_highwater_marks
-build_test_db memsubsys1-7 {
-  PRAGMA page_size=4096;
-  PRAGMA cache_size=10;
-  PRAGMA temp_store=memory;
-}
-#show_memstats
-do_test memsubsys1-7.3 {
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-  expr {$pg_used<24}
-} 1
-do_test memsubsys1-7.4 {
-  set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
-} 0
-do_test memsubsys1-7.5 {
-  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-  expr {$maxreq<(4100 + 8200*[nonzero_reserved_bytes])}
-} 1
-do_test memsubsys1-7.6 {
-  set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
-} 1
-do_test memsubsys1-7.7 {
-  set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]
-} 0
-
-# Test 8:  Disable PAGECACHE.  Make available SCRATCH zero.  Verify that
-# the SCRATCH overflow logic works.
-#
-db close
-sqlite3_shutdown
-sqlite3_config_pagecache 0 0
-sqlite3_config_scratch 25000 0
-sqlite3_initialize
-reset_highwater_marks
-do_test memsubsys1-8.1 {
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
-} 0
-do_test memsubsys1-8.2 {
-  set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]
-} 0
-do_test memsubsys1-8.3 {
-  sqlite3 db :memory:
-  db eval {
-    CREATE TABLE t1(x);
-    INSERT INTO t1 VALUES(zeroblob(400));
-    INSERT INTO t1 VALUES(zeroblob(400));
-    INSERT INTO t1 SELECT * FROM t1;
-    INSERT INTO t1 SELECT * FROM t1;
-    INSERT INTO t1 SELECT * FROM t1;
-  }
-  expr {[lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]>0}
-} 1
-db close
-sqlite3_shutdown
-sqlite3_config_memstatus 0
-sqlite3_initialize
-do_test memsubsys1-8.4 {
-  sqlite3 db :memory:
-  db eval {
-    CREATE TABLE t1(x);
-    INSERT INTO t1 VALUES(zeroblob(400));
-    INSERT INTO t1 VALUES(zeroblob(400));
-    INSERT INTO t1 SELECT * FROM t1;
-    INSERT INTO t1 SELECT * FROM t1;
-    INSERT INTO t1 SELECT * FROM t1;
-    SELECT rowid FROM t1;
-  }
-} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
-
 
 db close
 sqlite3_shutdown
 sqlite3_config_memstatus 1
-sqlite3_config_scratch 0 0
 sqlite3_config_lookaside 100 500
 sqlite3_config serialized
 sqlite3_initialize
diff --git a/test/permutations.test b/test/permutations.test
index bcd06c14b7..5afc51cb7d 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -455,33 +455,31 @@ lappend ::testsuitelist xxx
 # Define the permutation test suites:
 #
 
-# Run some tests using pre-allocated page and scratch blocks.
+# Run some tests using pre-allocated page blocks.
 #
 # mmap1.test is excluded because a good number of its tests depend on 
 # the page-cache being larger than the database. But this permutation
 # causes the effective limit on the page-cache to be just 24 pages.
 #
 test_suite "memsubsys1" -description {
-  Tests using pre-allocated page and scratch blocks
+  Tests using pre-allocated page blocks
 } -files [
   test_set $::allquicktests -exclude ioerr5.test malloc5.test mmap1.test
 ] -initialize {
   test_set_config_pagecache 4096 24
   catch {db close}
   sqlite3_shutdown
-  sqlite3_config_scratch 25000 1
   sqlite3_initialize
   autoinstall_test_functions
 } -shutdown {
   test_restore_config_pagecache
   catch {db close}
   sqlite3_shutdown
-  sqlite3_config_scratch 0 0
   sqlite3_initialize
   autoinstall_test_functions
 }
 
-# Run some tests using pre-allocated page and scratch blocks. This time
+# Run some tests using pre-allocated page blocks. This time
 # the allocations are too small to use in most cases.
 #
 # Both ioerr5.test and malloc5.test are excluded because they test the
@@ -489,21 +487,19 @@ test_suite "memsubsys1" -description {
 # This functionality is disabled if a pre-allocated page block is provided.
 #
 test_suite "memsubsys2" -description {
-  Tests using small pre-allocated page and scratch blocks
+  Tests using small pre-allocated page blocks
 } -files [
   test_set $::allquicktests -exclude ioerr5.test malloc5.test
 ] -initialize {
   test_set_config_pagecache 512 5
   catch {db close}
   sqlite3_shutdown
-  sqlite3_config_scratch 1000 1
   sqlite3_initialize
   autoinstall_test_functions
 } -shutdown {
   test_restore_config_pagecache
   catch {db close}
   sqlite3_shutdown
-  sqlite3_config_scratch 0 0
   sqlite3_initialize
   autoinstall_test_functions
 }
diff --git a/test/speedtest1.c b/test/speedtest1.c
index db3a558a38..e374c22c1e 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -25,7 +25,6 @@ static const char zHelp[] =
   "  --primarykey        Use PRIMARY KEY instead of UNIQUE where appropriate\n"
   "  --repeat N          Repeat each SELECT N times (default: 1)\n"
   "  --reprepare         Reprepare each statement upon every invocation\n"
-  "  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
   "  --serialized        Set serialized threading mode\n"
   "  --singlethread      Set single-threaded mode - disables all mutexing\n"
   "  --sqlonly           No-op.  Only show the SQL that would have been run.\n"
@@ -1649,7 +1648,6 @@ int main(int argc, char **argv){
   int pageSize = 0;             /* Desired page size.  0 means default */
   int nPCache = 0, szPCache = 0;/* --pcache configuration */
   int doPCache = 0;             /* True if --pcache is seen */
-  int nScratch = 0, szScratch=0;/* --scratch configuration */
   int showStats = 0;            /* True for --stats */
   int nThread = 0;              /* --threads value */
   int mmapSize = 0;             /* How big of a memory map to use */
@@ -1661,7 +1659,6 @@ int main(int argc, char **argv){
   void *pHeap = 0;              /* Allocated heap space */
   void *pLook = 0;              /* Allocated lookaside space */
   void *pPCache = 0;            /* Allocated storage for pcache */
-  void *pScratch = 0;           /* Allocated storage for scratch */
   int iCur, iHi;                /* Stats values, current and "highwater" */
   int i;                        /* Loop counter */
   int rc;                       /* API return code */
@@ -1741,11 +1738,6 @@ int main(int argc, char **argv){
         i += 1;
       }else if( strcmp(z,"reprepare")==0 ){
         g.bReprepare = 1;
-      }else if( strcmp(z,"scratch")==0 ){
-        if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]);
-        nScratch = integerValue(argv[i+1]);
-        szScratch = integerValue(argv[i+2]);
-        i += 2;
 #if SQLITE_VERSION_NUMBER>=3006000
       }else if( strcmp(z,"serialized")==0 ){
         sqlite3_config(SQLITE_CONFIG_SERIALIZED);
@@ -1816,13 +1808,6 @@ int main(int argc, char **argv){
     rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache);
     if( rc ) fatal_error("pcache configuration failed: %d\n", rc);
   }
-  if( nScratch>0 && szScratch>0 ){
-    pScratch = malloc( nScratch*(sqlite3_int64)szScratch );
-    if( pScratch==0 ) fatal_error("cannot allocate %lld-byte scratch\n",
-                                 nScratch*(sqlite3_int64)szScratch);
-    rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch);
-    if( rc ) fatal_error("scratch configuration failed: %d\n", rc);
-  }
   if( nLook>=0 ){
     sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);
   }
@@ -1939,14 +1924,10 @@ int main(int argc, char **argv){
 #endif
     sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHi, 0);
     printf("-- Pcache Overflow Bytes:       %d (max %d)\n", iCur,iHi);
-    sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHi, 0);
-    printf("-- Scratch Overflow Bytes:      %d (max %d)\n", iCur,iHi);
     sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHi, 0);
     printf("-- Largest Allocation:          %d bytes\n",iHi);
     sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHi, 0);
     printf("-- Largest Pcache Allocation:   %d bytes\n",iHi);
-    sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHi, 0);
-    printf("-- Largest Scratch Allocation:  %d bytes\n", iHi);
   }
 #endif
 
@@ -1959,7 +1940,6 @@ int main(int argc, char **argv){
   /* Release memory */
   free( pLook );
   free( pPCache );
-  free( pScratch );
   free( pHeap );
   return 0;
 }
diff --git a/test/tester.tcl b/test/tester.tcl
index 38c19701c6..10a20a47d6 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -1241,14 +1241,6 @@ proc show_memstats {} {
   set x [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0]
   set val [format {now %10d  max %10d} [lindex $x 1] [lindex $x 2]]
   output1 "Page-cache overflow:  $val"
-  set x [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0]
-  set val [format {now %10d  max %10d} [lindex $x 1] [lindex $x 2]]
-  output1 "Scratch memory used:  $val"
-  set x [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0]
-  set y [sqlite3_status SQLITE_STATUS_SCRATCH_SIZE 0]
-  set val [format {now %10d  max %10d  max-size %10d} \
-               [lindex $x 1] [lindex $x 2] [lindex $y 2]]
-  output1 "Scratch overflow:     $val"
   ifcapable yytrackmaxstackdepth {
     set x [sqlite3_status SQLITE_STATUS_PARSER_STACK 0]
     set val [format {               max %10d} [lindex $x 2]]
diff --git a/test/wordcount.c b/test/wordcount.c
index bc1d499f15..27aed72460 100644
--- a/test/wordcount.c
+++ b/test/wordcount.c
@@ -633,14 +633,10 @@ int main(int argc, char **argv){
     printf("%s Outstanding Allocations:     %d (max %d)\n",zTag,iCur,iHiwtr);
     sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, 0);
     printf("%s Pcache Overflow Bytes:       %d (max %d)\n",zTag,iCur,iHiwtr);
-    sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, 0);
-    printf("%s Scratch Overflow Bytes:      %d (max %d)\n",zTag,iCur,iHiwtr);
     sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, 0);
     printf("%s Largest Allocation:          %d bytes\n",zTag,iHiwtr);
     sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, 0);
     printf("%s Largest Pcache Allocation:   %d bytes\n",zTag,iHiwtr);
-    sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, 0);
-    printf("%s Largest Scratch Allocation:  %d bytes\n",zTag,iHiwtr);
   }
   return 0;
 }

From 52df6f5e5cb421150b8ab9ad1ee51c9628e9ee2b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 28 Aug 2017 16:11:05 +0000
Subject: [PATCH 053/270] Avoid unnecessary mutexes in the pcache1
 implementation in the common case where no auxiliary page cache memory is
 configured.

FossilOrigin-Name: 1ba051e34d7512ab5e8fc969c1b5aaaf827b8e6493ba4235895257aca78b500f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache1.c |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 2131ba9ff1..3b3666f8f6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\srarely-used\sscratch\smemory\sallocator.\s\sThis\smakes\sthe\scode\ssmaller,\nfaster,\sand\seasier\sto\smaintain.\s\sIn\splace\sof\sthe\sscratch\sallocator,\sadd\sthe\nSQLITE_CONFIG_SMALL_MALLOC\sconfiguration\soption\sthat\sprovides\sa\shint\sto\sSQLite\nthat\slarge\smemory\sallocations\sshould\sbe\savoided.
-D 2017-08-28T15:51:35.625
+C Avoid\sunnecessary\smutexes\sin\sthe\spcache1\simplementation\sin\sthe\scommon\scase\nwhere\sno\sauxiliary\spage\scache\smemory\sis\sconfigured.
+D 2017-08-28T16:11:05.724
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -448,7 +448,7 @@ F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
-F src/pcache1.c 0b793738b5dddaf0a645784835c6b5557b1ecfaee339af9c26810c6ecdb273aa
+F src/pcache1.c ad5bc71727c2e825dcbf342413e1b4b09ed8520cd83903671e8bd03bc30b4c98
 F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2348045fbc3fc99e2b46095cc86db99815cd1f9254d30a3b72c2b15c02076a84
-R dbf94ead9bb02368db4853c4bef6223c
+P 54b000246cfb5c7b8adb61a17357ef5a49adddde9e48e8937834d5ba0beb8a6b
+R 83da5bfb65f5a08c38f889ed88d6f498
 U drh
-Z 970c3285dea1ce91571a8f403e94ff33
+Z cc59cd24e10bd66953b2a2efefd623c5
diff --git a/manifest.uuid b/manifest.uuid
index f6193d5c82..ceacf0dc17 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-54b000246cfb5c7b8adb61a17357ef5a49adddde9e48e8937834d5ba0beb8a6b
\ No newline at end of file
+1ba051e34d7512ab5e8fc969c1b5aaaf827b8e6493ba4235895257aca78b500f
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index 9c59574ace..7a19bd9674 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -250,6 +250,7 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
   if( pcache1.isInit ){
     PgFreeslot *p;
     if( pBuf==0 ) sz = n = 0;
+    if( n==0 ) sz = 0;
     sz = ROUNDDOWN8(sz);
     pcache1.szSlot = sz;
     pcache1.nSlot = pcache1.nFreeSlot = n;

From 83bebddbd91ea60568aa4be7c268e25b8a179e82 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 28 Aug 2017 17:00:12 +0000
Subject: [PATCH 054/270] Add the --enable-update-limit option to the
 ./configure script.

FossilOrigin-Name: 64a8ae68381b7fbb29b659901ca7ce8d50510e4753758d5761f7e41539288cef
---
 configure     | 44 ++++++++++++++++++++++++++++++--------------
 configure.ac  | 10 ++++++++++
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 4 files changed, 48 insertions(+), 22 deletions(-)

diff --git a/configure b/configure
index 7a81d52156..ed9ed5ffd4 100755
--- a/configure
+++ b/configure
@@ -909,6 +909,7 @@ enable_fts3
 enable_fts4
 enable_fts5
 enable_json1
+enable_update_limit
 enable_rtree
 enable_session
 enable_gcov
@@ -1560,6 +1561,7 @@ Optional Features:
   --enable-fts4           Enable the FTS4 extension
   --enable-fts5           Enable the FTS5 extension
   --enable-json1          Enable the JSON1 extension
+  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
   --enable-rtree          Enable the RTREE extension
   --enable-session        Enable the SESSION extension
   --enable-gcov           Enable coverage testing using gcov
@@ -3929,13 +3931,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3938: output\"" >&5)
+  (eval echo "\"\$as_me:3940: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5141,7 +5143,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5144 "configure"' > conftest.$ac_ext
+  echo '#line 5146 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6666,11 +6668,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6673: \$? = $ac_status" >&5
+   echo "$as_me:6675: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7005,11 +7007,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7012: \$? = $ac_status" >&5
+   echo "$as_me:7014: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7110,11 +7112,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7117: \$? = $ac_status" >&5
+   echo "$as_me:7119: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7165,11 +7167,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7172: \$? = $ac_status" >&5
+   echo "$as_me:7174: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9545,7 +9547,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9548 "configure"
+#line 9550 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9641,7 +9643,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9644 "configure"
+#line 9646 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -11539,6 +11541,20 @@ if test "${enable_json1}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
 
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+# Check whether --enable-update-limit was given.
+if test "${enable_update_limit+set}" = set; then :
+  enableval=$enable_update_limit; enable_udlimit=yes
+else
+  enable_udlimit=no
+fi
+
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
+fi
+
 #########
 # See whether we should enable RTREE
 # Check whether --enable-rtree was given.
diff --git a/configure.ac b/configure.ac
index 4deee8ddcb..55ab0673ee 100644
--- a/configure.ac
+++ b/configure.ac
@@ -644,6 +644,16 @@ if test "${enable_json1}" = "yes" ; then
   OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 fi
 
+#########
+# See whether we should enable the LIMIT clause on UPDATE and DELETE
+# statements.
+AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
+      [Enable the UPDATE/DELETE LIMIT clause]),
+      [enable_udlimit=yes],[enable_udlimit=no])
+if test "${enable_udlimit}" = "yes" ; then
+  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
+fi
+
 #########
 # See whether we should enable RTREE
 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
diff --git a/manifest b/manifest
index 3b3666f8f6..c1fc26d824 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sunnecessary\smutexes\sin\sthe\spcache1\simplementation\sin\sthe\scommon\scase\nwhere\sno\sauxiliary\spage\scache\smemory\sis\sconfigured.
-D 2017-08-28T16:11:05.724
+C Add\sthe\s--enable-update-limit\soption\sto\sthe\s./configure\sscript.
+D 2017-08-28T17:00:12.028
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure 2500b432572804093c9a2516e2b16dbcec5d797ea1cd654915cfecd1d7a39ee3 x
-F configure.ac 13f45f02e6c51dd0e347315b5401c3f047712b7f79b7f35619115c23755afcff
+F configure e691ad9b505f1f47bc5d99be9e1d49b1be9037e9cb3821c9b14c63c3d413d055 x
+F configure.ac bb85c1c53e952c8c7078a2f147eba613e0128b8b6e7780d64758d8fb29bcc695
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html 1f8b8d4c9f5cfe40e679fee279cc9eb2da8e6eb74ad406028538d7864cc4b6cb
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 54b000246cfb5c7b8adb61a17357ef5a49adddde9e48e8937834d5ba0beb8a6b
-R 83da5bfb65f5a08c38f889ed88d6f498
+P 1ba051e34d7512ab5e8fc969c1b5aaaf827b8e6493ba4235895257aca78b500f
+R 7e1ed169d6d4a03a64afe06fcba95957
 U drh
-Z cc59cd24e10bd66953b2a2efefd623c5
+Z 13247c5e6bbeac642354740051347480
diff --git a/manifest.uuid b/manifest.uuid
index ceacf0dc17..09eebfb4e5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1ba051e34d7512ab5e8fc969c1b5aaaf827b8e6493ba4235895257aca78b500f
\ No newline at end of file
+64a8ae68381b7fbb29b659901ca7ce8d50510e4753758d5761f7e41539288cef
\ No newline at end of file

From 52fb8e194564a83028c8d1ccb0a159806fb63029 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 29 Aug 2017 20:21:12 +0000
Subject: [PATCH 055/270] Faster memory allocation from lookaside by not trying
 to keep track of the number of outstanding allocations, and rather computing
 that value only when requested.

FossilOrigin-Name: a06263f1efd2d45eac88b8d59e8fe8e458670fa3808c795feaa7f247fc36cbe9
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/build.c     | 11 +++++++----
 src/main.c      | 12 ++++++++----
 src/malloc.c    | 15 +++++++--------
 src/sqliteInt.h |  7 ++++---
 src/status.c    | 33 ++++++++++++++++++++++++++++++---
 7 files changed, 67 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index c1fc26d824..a460f66c1f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s--enable-update-limit\soption\sto\sthe\s./configure\sscript.
-D 2017-08-28T17:00:12.028
+C Faster\smemory\sallocation\sfrom\slookaside\sby\snot\strying\sto\skeep\strack\sof\sthe\nnumber\sof\soutstanding\sallocations,\sand\srather\scomputing\sthat\svalue\sonly\nwhen\srequested.
+D 2017-08-29T20:21:12.167
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -402,7 +402,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 1033b88fe50aba7d364b5a19666a9a274caa8d4c25ab7f3914221997b46af44a
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c b47a0320c6d237718b8f493ac97d37d1071bce12aca668b15219187150c41295
+F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -421,8 +421,8 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 227a83d3f840d55e40360a1a8370c120f1466ea7d73b1fffb74b8f59ad0f4046
-F src/malloc.c e069cec00407e029d71fbc9440ebbb5833a629416324b592ade8fed93b045c83
+F src/main.c 48a641949ce76c857b8771230c4304501287285714ea796f803b4178629ed560
+F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -462,9 +462,9 @@ F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469
 F src/sqlite.h.in f18eef5b101d5f33f98ca43decb1f025c1b629f091ad77fe2190128e93938a5a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 9a283ecf57bb81e0040d243d56e91beae76f6d5762b3ac33c7f3ec6076a71d99
+F src/sqliteInt.h 60295f5f909e32aef1961075a8fa98df19335a4b7792b4a0b897f1d8789681c9
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
-F src/status.c 90450a491f3f59f6978ca9832023c31238f592064e405eeda971f3c0829564eb
+F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828
 F src/test1.c 8513b17ca4a7a9ba28748535d178b6e472ec7394ae0eea53907f2d3bcdbab2df
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1ba051e34d7512ab5e8fc969c1b5aaaf827b8e6493ba4235895257aca78b500f
-R 7e1ed169d6d4a03a64afe06fcba95957
+P 64a8ae68381b7fbb29b659901ca7ce8d50510e4753758d5761f7e41539288cef
+R bf9afa7c8e223eb9e939beb0ed77930c
 U drh
-Z 13247c5e6bbeac642354740051347480
+Z 4dae0b64b90b0b3580ad9499179d8a8f
diff --git a/manifest.uuid b/manifest.uuid
index 09eebfb4e5..9d0521f6c3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-64a8ae68381b7fbb29b659901ca7ce8d50510e4753758d5761f7e41539288cef
\ No newline at end of file
+a06263f1efd2d45eac88b8d59e8fe8e458670fa3808c795feaa7f247fc36cbe9
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 05126f6260..6cd23c2abf 100644
--- a/src/build.c
+++ b/src/build.c
@@ -598,13 +598,16 @@ void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){
 */
 static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
   Index *pIndex, *pNext;
-  TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */
 
+#ifdef SQLITE_DEBUG
   /* Record the number of outstanding lookaside allocations in schema Tables
   ** prior to doing any free() operations.  Since schema Tables do not use
   ** lookaside, this number should not change. */
-  TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ?
-                         db->lookaside.nOut : 0 );
+  int nLookaside = 0;
+  if( db && (pTable->tabFlags & TF_Ephemeral)==0 ){
+    nLookaside = sqlite3LookasideUsed(db, 0);
+  }
+#endif
 
   /* Delete all indices associated with this table. */
   for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){
@@ -638,7 +641,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
   sqlite3DbFree(db, pTable);
 
   /* Verify that no lookaside memory was used by schema tables */
-  assert( nLookaside==0 || nLookaside==db->lookaside.nOut );
+  assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) );
 }
 void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
   /* Do not delete the table until the reference count reaches zero. */
diff --git a/src/main.c b/src/main.c
index 212f7d6b5f..168ce851f6 100644
--- a/src/main.c
+++ b/src/main.c
@@ -658,7 +658,8 @@ int sqlite3_config(int op, ...){
 static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
 #ifndef SQLITE_OMIT_LOOKASIDE
   void *pStart;
-  if( db->lookaside.nOut ){
+  
+  if( sqlite3LookasideUsed(db,0)>0 ){
     return SQLITE_BUSY;
   }
   /* Free any existing lookaside buffer for this handle before
@@ -686,16 +687,18 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
     pStart = pBuf;
   }
   db->lookaside.pStart = pStart;
+  db->lookaside.pInit = 0;
   db->lookaside.pFree = 0;
   db->lookaside.sz = (u16)sz;
   if( pStart ){
     int i;
     LookasideSlot *p;
     assert( sz > (int)sizeof(LookasideSlot*) );
+    db->lookaside.nSlot = cnt;
     p = (LookasideSlot*)pStart;
     for(i=cnt-1; i>=0; i--){
-      p->pNext = db->lookaside.pFree;
-      db->lookaside.pFree = p;
+      p->pNext = db->lookaside.pInit;
+      db->lookaside.pInit = p;
       p = (LookasideSlot*)&((u8*)p)[sz];
     }
     db->lookaside.pEnd = p;
@@ -706,6 +709,7 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
     db->lookaside.pEnd = db;
     db->lookaside.bDisable = 1;
     db->lookaside.bMalloced = 0;
+    db->lookaside.nSlot = 0;
   }
 #endif /* SQLITE_OMIT_LOOKASIDE */
   return SQLITE_OK;
@@ -1225,7 +1229,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){
   sqlite3_mutex_leave(db->mutex);
   db->magic = SQLITE_MAGIC_CLOSED;
   sqlite3_mutex_free(db->mutex);
-  assert( db->lookaside.nOut==0 );  /* Fails on a lookaside memory leak */
+  assert( sqlite3LookasideUsed(db,0)==0 );
   if( db->lookaside.bMalloced ){
     sqlite3_free(db->lookaside.pStart);
   }
diff --git a/src/malloc.c b/src/malloc.c
index 78a50b49c5..b750f6e72f 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -354,7 +354,6 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){
 #endif
       pBuf->pNext = db->lookaside.pFree;
       db->lookaside.pFree = pBuf;
-      db->lookaside.nOut--;
       return;
     }
   }
@@ -515,16 +514,16 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
     assert( db->mallocFailed==0 );
     if( n>db->lookaside.sz ){
       db->lookaside.anStat[1]++;
-    }else if( (pBuf = db->lookaside.pFree)==0 ){
-      db->lookaside.anStat[2]++;
-    }else{
+    }else if( (pBuf = db->lookaside.pFree)!=0 ){
       db->lookaside.pFree = pBuf->pNext;
-      db->lookaside.nOut++;
       db->lookaside.anStat[0]++;
-      if( db->lookaside.nOut>db->lookaside.mxOut ){
-        db->lookaside.mxOut = db->lookaside.nOut;
-      }
       return (void*)pBuf;
+    }else if( (pBuf = db->lookaside.pInit)!=0 ){
+      db->lookaside.pInit = pBuf->pNext;
+      db->lookaside.anStat[0]++;
+      return (void*)pBuf;
+    }else{
+      db->lookaside.anStat[2]++;
     }
   }else if( db->mallocFailed ){
     return 0;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 55a8efe214..1c1879a922 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1245,9 +1245,9 @@ struct Lookaside {
   u32 bDisable;           /* Only operate the lookaside when zero */
   u16 sz;                 /* Size of each buffer in bytes */
   u8 bMalloced;           /* True if pStart obtained from sqlite3_malloc() */
-  int nOut;               /* Number of buffers currently checked out */
-  int mxOut;              /* Highwater mark for nOut */
-  int anStat[3];          /* 0: hits.  1: size misses.  2: full misses */
+  u32 nSlot;              /* Number of lookaside slots allocated */
+  u32 anStat[3];          /* 0: hits.  1: size misses.  2: full misses */
+  LookasideSlot *pInit;   /* List of buffers not previously used */
   LookasideSlot *pFree;   /* List of available buffers */
   void *pStart;           /* First byte of available memory space */
   void *pEnd;             /* First byte past end of available space */
@@ -3575,6 +3575,7 @@ sqlite3_int64 sqlite3StatusValue(int);
 void sqlite3StatusUp(int, int);
 void sqlite3StatusDown(int, int);
 void sqlite3StatusHighwater(int, int);
+int sqlite3LookasideUsed(sqlite3*,int*);
 
 /* Access to mutexes used by sqlite3_status() */
 sqlite3_mutex *sqlite3Pcache1Mutex(void);
diff --git a/src/status.c b/src/status.c
index b783b395d3..6e5b0e573b 100644
--- a/src/status.c
+++ b/src/status.c
@@ -170,6 +170,28 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){
   return rc;
 }
 
+/*
+** Return the number of LookasideSlot elements on the linked list
+*/
+static u32 countLookasideSlots(LookasideSlot *p){
+  u32 cnt = 0;
+  while( p ){
+    p = p->pNext;
+    cnt++;
+  }
+  return cnt;
+}
+
+/*
+** Count the number of slots of lookaside memory that are outstanding
+*/
+int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){
+  u32 nInit = countLookasideSlots(db->lookaside.pInit);
+  u32 nFree = countLookasideSlots(db->lookaside.pFree);
+  if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit;
+  return db->lookaside.nSlot - (nInit+nFree);
+}
+
 /*
 ** Query status information for a single database connection
 */
@@ -189,10 +211,15 @@ int sqlite3_db_status(
   sqlite3_mutex_enter(db->mutex);
   switch( op ){
     case SQLITE_DBSTATUS_LOOKASIDE_USED: {
-      *pCurrent = db->lookaside.nOut;
-      *pHighwater = db->lookaside.mxOut;
+      *pCurrent = sqlite3LookasideUsed(db, pHighwater);
       if( resetFlag ){
-        db->lookaside.mxOut = db->lookaside.nOut;
+        LookasideSlot *p = db->lookaside.pFree;
+        if( p ){
+          while( p->pNext ) p = p->pNext;
+          p->pNext = db->lookaside.pInit;
+          db->lookaside.pInit = db->lookaside.pFree;
+          db->lookaside.pFree = 0;
+        }
       }
       break;
     }

From 617b7b42e3899b549895b6f09cbb8a5b527ca06f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 30 Aug 2017 04:44:59 +0000
Subject: [PATCH 056/270] Small performance optimization in pcache1.

FossilOrigin-Name: ffd437da9541f8a2792e3e07c0a43f388f856fdc211fe42755eb51bfa5995d9f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache1.c | 26 ++++++++++++++------------
 3 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index a460f66c1f..236c5cdbba 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Faster\smemory\sallocation\sfrom\slookaside\sby\snot\strying\sto\skeep\strack\sof\sthe\nnumber\sof\soutstanding\sallocations,\sand\srather\scomputing\sthat\svalue\sonly\nwhen\srequested.
-D 2017-08-29T20:21:12.167
+C Small\sperformance\soptimization\sin\spcache1.
+D 2017-08-30T04:44:59.152
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -448,7 +448,7 @@ F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
 F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
-F src/pcache1.c ad5bc71727c2e825dcbf342413e1b4b09ed8520cd83903671e8bd03bc30b4c98
+F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64a8ae68381b7fbb29b659901ca7ce8d50510e4753758d5761f7e41539288cef
-R bf9afa7c8e223eb9e939beb0ed77930c
+P a06263f1efd2d45eac88b8d59e8fe8e458670fa3808c795feaa7f247fc36cbe9
+R bcdeb031d82a94a75572b4604ebe0254
 U drh
-Z 4dae0b64b90b0b3580ad9499179d8a8f
+Z 5debc6036b056965ebab42f5ae8d1e01
diff --git a/manifest.uuid b/manifest.uuid
index 9d0521f6c3..f8c62cd0d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a06263f1efd2d45eac88b8d59e8fe8e458670fa3808c795feaa7f247fc36cbe9
\ No newline at end of file
+ffd437da9541f8a2792e3e07c0a43f388f856fdc211fe42755eb51bfa5995d9f
\ No newline at end of file
diff --git a/src/pcache1.c b/src/pcache1.c
index 7a19bd9674..fc3cbc5abe 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -137,7 +137,7 @@ struct PGroup {
   unsigned int nMaxPage;         /* Sum of nMax for purgeable caches */
   unsigned int nMinPage;         /* Sum of nMin for purgeable caches */
   unsigned int mxPinned;         /* nMaxpage + 10 - nMinPage */
-  unsigned int nCurrentPage;     /* Number of purgeable pages allocated */
+  unsigned int nPurgeable;       /* Number of purgeable pages allocated */
   PgHdr1 lru;                    /* The beginning and end of the LRU list */
 };
 
@@ -151,11 +151,13 @@ struct PGroup {
 */
 struct PCache1 {
   /* Cache configuration parameters. Page size (szPage) and the purgeable
-  ** flag (bPurgeable) are set when the cache is created. nMax may be 
+  ** flag (bPurgeable) and the pnPurgeable pointer are all set when the
+  ** cache is created and are never changed thereafter. nMax may be 
   ** modified at any time by a call to the pcache1Cachesize() method.
   ** The PGroup mutex must be held when accessing nMax.
   */
   PGroup *pGroup;                     /* PGroup this cache belongs to */
+  unsigned int *pnPurgeable;          /* Pointer to pGroup->nPurgeable */
   int szPage;                         /* Size of database content section */
   int szExtra;                        /* sizeof(MemPage)+sizeof(PgHdr) */
   int szAlloc;                        /* Total size of one pcache line */
@@ -443,9 +445,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){
     p->isBulkLocal = 0;
     p->isAnchor = 0;
   }
-  if( pCache->bPurgeable ){
-    pCache->pGroup->nCurrentPage++;
-  }
+  (*pCache->pnPurgeable)++;
   return p;
 }
 
@@ -466,9 +466,7 @@ static void pcache1FreePage(PgHdr1 *p){
     sqlite3_free(p);
 #endif
   }
-  if( pCache->bPurgeable ){
-    pCache->pGroup->nCurrentPage--;
-  }
+  (*pCache->pnPurgeable)--;
 }
 
 /*
@@ -608,7 +606,7 @@ static void pcache1EnforceMaxPage(PCache1 *pCache){
   PGroup *pGroup = pCache->pGroup;
   PgHdr1 *p;
   assert( sqlite3_mutex_held(pGroup->mutex) );
-  while( pGroup->nCurrentPage>pGroup->nMaxPage
+  while( pGroup->nPurgeable>pGroup->nMaxPage
       && (p=pGroup->lru.pLruPrev)->isAnchor==0
   ){
     assert( p->pCache->pGroup==pGroup );
@@ -779,6 +777,10 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
       pCache->nMin = 10;
       pGroup->nMinPage += pCache->nMin;
       pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
+      pCache->pnPurgeable = &pGroup->nPurgeable;
+    }else{
+      static unsigned int dummyCurrentPage;
+      pCache->pnPurgeable = &dummyCurrentPage;
     }
     pcache1LeaveMutex(pGroup);
     if( pCache->nHash==0 ){
@@ -888,7 +890,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2(
       pcache1FreePage(pPage);
       pPage = 0;
     }else{
-      pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable);
+      pGroup->nPurgeable -= (pOther->bPurgeable - pCache->bPurgeable);
     }
   }
 
@@ -1069,7 +1071,7 @@ static void pcache1Unpin(
   assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
   assert( PAGE_IS_PINNED(pPage) );
 
-  if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){
+  if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
     pcache1RemoveFromHash(pPage, 1);
   }else{
     /* Add the page to the PGroup LRU list. */
@@ -1248,7 +1250,7 @@ void sqlite3PcacheStats(
     assert( PAGE_IS_UNPINNED(p) );
     nRecyclable++;
   }
-  *pnCurrent = pcache1.grp.nCurrentPage;
+  *pnCurrent = pcache1.grp.nPurgeable;
   *pnMax = (int)pcache1.grp.nMaxPage;
   *pnMin = (int)pcache1.grp.nMinPage;
   *pnRecyclable = nRecyclable;

From 9c1cf3235c6e0ddef741c3dd9fdf2af076484a59 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 30 Aug 2017 13:21:17 +0000
Subject: [PATCH 057/270] Remove unnecessary "__declspec(dllexport)" qualifiers
 from generated file shell.c.

FossilOrigin-Name: bcc20be5b290c563183e82a590cc1fdabadfb13475fd8f6b3d810365cea5d868
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/shell.c       | 44 ++++++++++----------------------------------
 tool/mkshellc.tcl |  1 +
 4 files changed, 20 insertions(+), 43 deletions(-)

diff --git a/manifest b/manifest
index 236c5cdbba..f4fac0921e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\sperformance\soptimization\sin\spcache1.
-D 2017-08-30T04:44:59.152
+C Remove\sunnecessary\s"__declspec(dllexport)"\squalifiers\sfrom\sgenerated\sfile\nshell.c.
+D 2017-08-30T13:21:17.718
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -457,7 +457,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
-F src/shell.c 117305aab365a0448505d8cfcc27d58b0182ea314f0201bd26c340a5717419a4
+F src/shell.c 319082accd4b719a332f4cf92347546e310327b4b4d50c42de129787b5ed2946
 F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469c
 F src/sqlite.h.in f18eef5b101d5f33f98ca43decb1f025c1b629f091ad77fe2190128e93938a5a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1588,7 +1588,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 8743a62e12ab67741f63f3e8ea00c482f8fa50ae3d3bca16b38754641777bf13
+F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb3b0b
 F tool/mksourceid.c 30966d568654a4fd962fb324753e49429b7379e1f72d2be489ade963121f5943
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a06263f1efd2d45eac88b8d59e8fe8e458670fa3808c795feaa7f247fc36cbe9
-R bcdeb031d82a94a75572b4604ebe0254
-U drh
-Z 5debc6036b056965ebab42f5ae8d1e01
+P ffd437da9541f8a2792e3e07c0a43f388f856fdc211fe42755eb51bfa5995d9f
+R 62e1ab098327dc96140157e6ff15fb3c
+U dan
+Z 8f2a622e90379c693a565adc570bf8df
diff --git a/manifest.uuid b/manifest.uuid
index f8c62cd0d5..9e337ae556 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ffd437da9541f8a2792e3e07c0a43f388f856fdc211fe42755eb51bfa5995d9f
\ No newline at end of file
+bcc20be5b290c563183e82a590cc1fdabadfb13475fd8f6b3d810365cea5d868
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index b78da38701..a87aab2bd8 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -1499,7 +1499,7 @@ static void sha3QueryFunc(
 
 
 #ifdef _WIN32
-__declspec(dllexport)
+
 #endif
 int sqlite3_shathree_init(
   sqlite3 *db,
@@ -1611,7 +1611,7 @@ static void writefileFunc(
 
 
 #ifdef _WIN32
-__declspec(dllexport)
+
 #endif
 int sqlite3_fileio_init(
   sqlite3 *db, 
@@ -2139,7 +2139,7 @@ int sqlite3CompletionVtabInit(sqlite3 *db){
 }
 
 #ifdef _WIN32
-__declspec(dllexport)
+
 #endif
 int sqlite3_completion_init(
   sqlite3 *db, 
@@ -2231,14 +2231,13 @@ struct ShellState {
 /*
 ** These are the allowed shellFlgs values
 */
-#define SHFLG_Scratch        0x00000001 /* The --scratch option is used */
-#define SHFLG_Pagecache      0x00000002 /* The --pagecache option is used */
-#define SHFLG_Lookaside      0x00000004 /* Lookaside memory is used */
-#define SHFLG_Backslash      0x00000008 /* The --backslash option is used */
-#define SHFLG_PreserveRowid  0x00000010 /* .dump preserves rowid values */
-#define SHFLG_Newlines       0x00000020 /* .dump --newline flag */
-#define SHFLG_CountChanges   0x00000040 /* .changes setting */
-#define SHFLG_Echo           0x00000080 /* .echo or --echo setting */
+#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
+#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
+#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
+#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
+#define SHFLG_Newlines       0x00000010 /* .dump --newline flag */
+#define SHFLG_CountChanges   0x00000020 /* .changes setting */
+#define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
 
 /*
 ** Macros for testing and setting shellFlgs
@@ -3257,18 +3256,10 @@ static int display_stats(
     }
     displayStatLine(pArg, "Number of Pcache Overflow Bytes:",
        "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset);
-    if( pArg->shellFlgs & SHFLG_Scratch ){
-      displayStatLine(pArg, "Number of Scratch Allocations Used:",
-         "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset);
-    }
-    displayStatLine(pArg, "Number of Scratch Overflow Bytes:",
-       "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset);
     displayStatLine(pArg, "Largest Allocation:",
        "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset);
     displayStatLine(pArg, "Largest Pcache Allocation:",
        "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset);
-    displayStatLine(pArg, "Largest Scratch Allocation:",
-       "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset);
 #ifdef YYTRACKMAXSTACKDEPTH
     displayStatLine(pArg, "Deepest Parser Stack:",
        "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset);
@@ -7289,7 +7280,6 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "reserve",               SQLITE_TESTCTRL_RESERVE                },
       { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
       { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
-      { "scratchmalloc",         SQLITE_TESTCTRL_SCRATCHMALLOC          },
       { "byteorder",             SQLITE_TESTCTRL_BYTEORDER              },
       { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
       { "imposter",              SQLITE_TESTCTRL_IMPOSTER               },
@@ -7402,7 +7392,6 @@ static int do_meta_command(char *zLine, ShellState *p){
         case SQLITE_TESTCTRL_BITVEC_TEST:
         case SQLITE_TESTCTRL_FAULT_INSTALL:
         case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS:
-        case SQLITE_TESTCTRL_SCRATCHMALLOC:
         default:
           utf8_printf(stderr,
                       "Error: CLI support for testctrl %s not implemented\n",
@@ -7922,7 +7911,6 @@ static const char zOptions[] =
   "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
   "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
   "   -quote               set output mode to 'quote'\n"
-  "   -scratch SIZE N      use N slots of SZ bytes each for scratch memory\n"
   "   -separator SEP       set output column separator. Default: '|'\n"
   "   -stats               print memory stats before each finalize\n"
   "   -version             show SQLite version\n"
@@ -8120,16 +8108,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
       (void)cmdline_option_value(argc, argv, ++i);
 #endif
-    }else if( strcmp(z,"-scratch")==0 ){
-      int n, sz;
-      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( sz>400000 ) sz = 400000;
-      if( sz<2500 ) sz = 2500;
-      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( n>10 ) n = 10;
-      if( n<1 ) n = 1;
-      sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n);
-      data.shellFlgs |= SHFLG_Scratch;
     }else if( strcmp(z,"-pagecache")==0 ){
       int n, sz;
       sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
@@ -8273,8 +8251,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       stdin_is_interactive = 0;
     }else if( strcmp(z,"-heap")==0 ){
       i++;
-    }else if( strcmp(z,"-scratch")==0 ){
-      i+=2;
     }else if( strcmp(z,"-pagecache")==0 ){
       i+=2;
     }else if( strcmp(z,"-lookaside")==0 ){
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index aef127e581..7ce9dc4431 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -35,6 +35,7 @@ while {1} {
     while {![eof $in2]} {
       set lx [gets $in2]
       if {[regexp {^#include "sqlite} $lx]} continue
+      set lx [string map [list __declspec(dllexport) {}] $lx]
       puts $out $lx
     }
     close $in2

From 38688b0b1da8881c1c7dc31659eb09bba92c2b76 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 31 Aug 2017 21:11:52 +0000
Subject: [PATCH 058/270] Clarify the documentation about "protected" versus
 "unprotected" sqlite3_value objects.  No changes to code.

FossilOrigin-Name: 4094a3410e05597d0adc740d161e0358a3041c5a4e73bde35b1e983594c6879a
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  5 +++--
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index f4fac0921e..e5ab14dd3b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sunnecessary\s"__declspec(dllexport)"\squalifiers\sfrom\sgenerated\sfile\nshell.c.
-D 2017-08-30T13:21:17.718
+C Clarify\sthe\sdocumentation\sabout\s"protected"\sversus\s"unprotected"\ssqlite3_value\nobjects.\s\sNo\schanges\sto\scode.
+D 2017-08-31T21:11:52.089
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -459,7 +459,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c 319082accd4b719a332f4cf92347546e310327b4b4d50c42de129787b5ed2946
 F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469c
-F src/sqlite.h.in f18eef5b101d5f33f98ca43decb1f025c1b629f091ad77fe2190128e93938a5a
+F src/sqlite.h.in 21f62793a3611b43f6fb31f0a4c8b38489a4df025416e9b7db7cc01bf5ef5aaa
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h 60295f5f909e32aef1961075a8fa98df19335a4b7792b4a0b897f1d8789681c9
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ffd437da9541f8a2792e3e07c0a43f388f856fdc211fe42755eb51bfa5995d9f
-R 62e1ab098327dc96140157e6ff15fb3c
-U dan
-Z 8f2a622e90379c693a565adc570bf8df
+P bcc20be5b290c563183e82a590cc1fdabadfb13475fd8f6b3d810365cea5d868
+R d2d3bdf1e274f5eba4292da390f3324b
+U drh
+Z c0deaaf09b7cbc0b0d0b890200390003
diff --git a/manifest.uuid b/manifest.uuid
index 9e337ae556..c3f1122792 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bcc20be5b290c563183e82a590cc1fdabadfb13475fd8f6b3d810365cea5d868
\ No newline at end of file
+4094a3410e05597d0adc740d161e0358a3041c5a4e73bde35b1e983594c6879a
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index fd30eab9a3..8b3c22c92d 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3819,8 +3819,9 @@ int sqlite3_stmt_busy(sqlite3_stmt*);
 ** implementation of [application-defined SQL functions] are protected.
 ** ^The sqlite3_value object returned by
 ** [sqlite3_column_value()] is unprotected.
-** Unprotected sqlite3_value objects may only be used with
-** [sqlite3_result_value()] and [sqlite3_bind_value()].
+** Unprotected sqlite3_value objects may only be used as arguments
+** to [sqlite3_result_value()], [sqlite3_bind_value()], and
+** [sqlite3_value_dup()].
 ** The [sqlite3_value_blob | sqlite3_value_type()] family of
 ** interfaces require protected sqlite3_value objects.
 */

From f0dae6d0a276566e384d38244b58ceda3ae58907 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 1 Sep 2017 12:18:41 +0000
Subject: [PATCH 059/270] Small size and performance improvement in
 pcacheManageDirtyList() by not zeroing the PgHdr.pDirtyNext and
 PgHdr.pDirtyPrev pointers for PgHdr objects that are not on the dirty list.

FossilOrigin-Name: 919863b14859d958d6c078097faae02070c7bd082e2814cf3f84bd84921e4419
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/pcache.c  |  5 +----
 src/pcache.h  |  2 ++
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index e5ab14dd3b..8eef081e9b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clarify\sthe\sdocumentation\sabout\s"protected"\sversus\s"unprotected"\ssqlite3_value\nobjects.\s\sNo\schanges\sto\scode.
-D 2017-08-31T21:11:52.089
+C Small\ssize\sand\sperformance\simprovement\sin\spcacheManageDirtyList()\sby\snot\nzeroing\sthe\sPgHdr.pDirtyNext\sand\sPgHdr.pDirtyPrev\spointers\sfor\sPgHdr\sobjects\nthat\sare\snot\son\sthe\sdirty\slist.
+D 2017-09-01T12:18:41.125
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -446,8 +446,8 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
-F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
-F src/pcache.h 521bb9610d38ef17a3cc9b5ddafd4546c2ea67fa3d0e464823d73c2a28d50e11
+F src/pcache.c f79181979fe7954b7bb4ef6c9689eabaa6733ed7393b25a4f77e15d11362b29d
+F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bcc20be5b290c563183e82a590cc1fdabadfb13475fd8f6b3d810365cea5d868
-R d2d3bdf1e274f5eba4292da390f3324b
+P 4094a3410e05597d0adc740d161e0358a3041c5a4e73bde35b1e983594c6879a
+R 5aa7325d41fae70be10866bcc032aad5
 U drh
-Z c0deaaf09b7cbc0b0d0b890200390003
+Z 3a2ed42dc8bf752f068ef19d1afbebba
diff --git a/manifest.uuid b/manifest.uuid
index c3f1122792..ff56ea75d9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4094a3410e05597d0adc740d161e0358a3041c5a4e73bde35b1e983594c6879a
\ No newline at end of file
+919863b14859d958d6c078097faae02070c7bd082e2814cf3f84bd84921e4419
\ No newline at end of file
diff --git a/src/pcache.c b/src/pcache.c
index dc7d00f306..9687ac770f 100644
--- a/src/pcache.c
+++ b/src/pcache.c
@@ -191,12 +191,9 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
         p->eCreate = 2;
       }
     }
-    pPage->pDirtyNext = 0;
-    pPage->pDirtyPrev = 0;
   }
   if( addRemove & PCACHE_DIRTYLIST_ADD ){
-    assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage );
-  
+    pPage->pDirtyPrev = 0;
     pPage->pDirtyNext = p->pDirty;
     if( pPage->pDirtyNext ){
       assert( pPage->pDirtyNext->pDirtyPrev==0 );
diff --git a/src/pcache.h b/src/pcache.h
index 864ce5bb4a..bbc2cb4539 100644
--- a/src/pcache.h
+++ b/src/pcache.h
@@ -43,6 +43,8 @@ struct PgHdr {
   i16 nRef;                      /* Number of users of this page */
   PgHdr *pDirtyNext;             /* Next element in list of dirty pages */
   PgHdr *pDirtyPrev;             /* Previous element in list of dirty pages */
+                          /* NB: pDirtyNext and pDirtyPrev are undefined if the
+                          ** PgHdr object is not dirty */
 };
 
 /* Bit values for PgHdr.flags */

From dfcdc663f27897b62c6b45fcab44424afaf5b408 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 1 Sep 2017 12:57:33 +0000
Subject: [PATCH 060/270] Remove an obsolete optimization in pcache that due to
 more recent changes was recently making the code a little slower.

FossilOrigin-Name: c4e7e175eecfd79015f4fae99618dfce6baf97c21bf3c909ea535d4e12dcaaad
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pcache.c  |  6 +-----
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 8eef081e9b..082580d908 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\ssize\sand\sperformance\simprovement\sin\spcacheManageDirtyList()\sby\snot\nzeroing\sthe\sPgHdr.pDirtyNext\sand\sPgHdr.pDirtyPrev\spointers\sfor\sPgHdr\sobjects\nthat\sare\snot\son\sthe\sdirty\slist.
-D 2017-09-01T12:18:41.125
+C Remove\san\sobsolete\soptimization\sin\spcache\sthat\sdue\sto\smore\srecent\schanges\nwas\srecently\smaking\sthe\scode\sa\slittle\sslower.
+D 2017-09-01T12:57:33.041
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -446,7 +446,7 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
-F src/pcache.c f79181979fe7954b7bb4ef6c9689eabaa6733ed7393b25a4f77e15d11362b29d
+F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4094a3410e05597d0adc740d161e0358a3041c5a4e73bde35b1e983594c6879a
-R 5aa7325d41fae70be10866bcc032aad5
+P 919863b14859d958d6c078097faae02070c7bd082e2814cf3f84bd84921e4419
+R 0eee553fc228f3f42e4580f46005e756
 U drh
-Z 3a2ed42dc8bf752f068ef19d1afbebba
+Z 71c472332c9be0857de95ba6da186b34
diff --git a/manifest.uuid b/manifest.uuid
index ff56ea75d9..e4cecf80a4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-919863b14859d958d6c078097faae02070c7bd082e2814cf3f84bd84921e4419
\ No newline at end of file
+c4e7e175eecfd79015f4fae99618dfce6baf97c21bf3c909ea535d4e12dcaaad
\ No newline at end of file
diff --git a/src/pcache.c b/src/pcache.c
index 9687ac770f..6f43a1e8af 100644
--- a/src/pcache.c
+++ b/src/pcache.c
@@ -510,11 +510,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
   if( (--p->nRef)==0 ){
     if( p->flags&PGHDR_CLEAN ){
       pcacheUnpin(p);
-    }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/
-      /* Move the page to the head of the dirty list. If p->pDirtyPrev==0,
-      ** then page p is already at the head of the dirty list and the
-      ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE
-      ** tag above.  */
+    }else{
       pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT);
     }
   }

From 3908fe90b1e9b0c146d6548f5c100613f88dc9d5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 1 Sep 2017 14:50:19 +0000
Subject: [PATCH 061/270] Add the new sqlite3PagerUnrefPageOne() pager method
 to deal with the special case of unreferencing page1.

FossilOrigin-Name: b2e2100cf766da1cb499aec0ca0a9969d7af5d93312ab8cf895fbf4a6ffb9b2f
---
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 src/btree.c   | 25 +++++++++++++++++++------
 src/pager.c   | 31 +++++++++++++++++++++++--------
 src/pager.h   |  1 +
 5 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index 082580d908..633c8bc712 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sobsolete\soptimization\sin\spcache\sthat\sdue\sto\smore\srecent\schanges\nwas\srecently\smaking\sthe\scode\sa\slittle\sslower.
-D 2017-09-01T12:57:33.041
+C Add\sthe\snew\ssqlite3PagerUnrefPageOne()\spager\smethod\sto\sdeal\swith\sthe\sspecial\ncase\sof\sunreferencing\spage1.
+D 2017-09-01T14:50:19.167
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 1033b88fe50aba7d364b5a19666a9a274caa8d4c25ab7f3914221997b46af44a
+F src/btree.c f1ac9509f4595d034fe432982c39d51ac2bfc9bd83136d16255aede35c526b45
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
@@ -443,8 +443,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5
-F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
+F src/pager.c 3936b32a0a45f1b80462b38ba1ed0556e62843554a089ab9c4be1a35c97f23bc
+F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 919863b14859d958d6c078097faae02070c7bd082e2814cf3f84bd84921e4419
-R 0eee553fc228f3f42e4580f46005e756
+P c4e7e175eecfd79015f4fae99618dfce6baf97c21bf3c909ea535d4e12dcaaad
+R 6f9a52521f107a3c330b7cfcb23523cf
 U drh
-Z 71c472332c9be0857de95ba6da186b34
+Z f39368718f6b0d67941da00fc11a34d5
diff --git a/manifest.uuid b/manifest.uuid
index e4cecf80a4..1aa72acac9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c4e7e175eecfd79015f4fae99618dfce6baf97c21bf3c909ea535d4e12dcaaad
\ No newline at end of file
+b2e2100cf766da1cb499aec0ca0a9969d7af5d93312ab8cf895fbf4a6ffb9b2f
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 9ec61507cb..3ca0037e9d 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -439,7 +439,8 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);
+static void releasePage(MemPage *pPage);         /* Forward reference */
+static void releasePageOne(MemPage *pPage);      /* Forward reference */
 static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
 /*
@@ -2109,6 +2110,8 @@ getAndInitPage_error:
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to btreeGetPage.
+**
+** Page1 is a special case and must be released using releasePageOne().
 */
 static void releasePageNotNull(MemPage *pPage){
   assert( pPage->aData );
@@ -2122,6 +2125,16 @@ static void releasePageNotNull(MemPage *pPage){
 static void releasePage(MemPage *pPage){
   if( pPage ) releasePageNotNull(pPage);
 }
+static void releasePageOne(MemPage *pPage){
+  assert( pPage!=0 );
+  assert( pPage->aData );
+  assert( pPage->pBt );
+  assert( pPage->pDbPage!=0 );
+  assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage );
+  assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData );
+  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
+  sqlite3PagerUnrefPageOne(pPage->pDbPage);
+}
 
 /*
 ** Get an unused page.
@@ -3000,7 +3013,7 @@ static int lockBtree(BtShared *pBt){
       }else{
         setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1);
         if( isOpen==0 ){
-          releasePage(pPage1);
+          releasePageOne(pPage1);
           return SQLITE_OK;
         }
       }
@@ -3047,7 +3060,7 @@ static int lockBtree(BtShared *pBt){
       ** zero and return SQLITE_OK. The caller will call this function
       ** again with the correct page-size.
       */
-      releasePage(pPage1);
+      releasePageOne(pPage1);
       pBt->usableSize = usableSize;
       pBt->pageSize = pageSize;
       freeTempSpace(pBt);
@@ -3101,7 +3114,7 @@ static int lockBtree(BtShared *pBt){
   return SQLITE_OK;
 
 page1_init_failed:
-  releasePage(pPage1);
+  releasePageOne(pPage1);
   pBt->pPage1 = 0;
   return rc;
 }
@@ -3146,7 +3159,7 @@ static void unlockBtreeIfUnused(BtShared *pBt){
     assert( pPage1->aData );
     assert( sqlite3PagerRefcount(pBt->pPager)==1 );
     pBt->pPage1 = 0;
-    releasePageNotNull(pPage1);
+    releasePageOne(pPage1);
   }
 }
 
@@ -4068,7 +4081,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){
       if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage);
       testcase( pBt->nPage!=nPage );
       pBt->nPage = nPage;
-      releasePage(pPage1);
+      releasePageOne(pPage1);
     }
     assert( countValidCursors(pBt, 1)==0 );
     pBt->inTransaction = TRANS_READ;
diff --git a/src/pager.c b/src/pager.c
index 2e5977fe43..b3ec03bd95 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -5350,7 +5350,8 @@ int sqlite3PagerSharedLock(Pager *pPager){
 ** nothing to rollback, so this routine is a no-op.
 */ 
 static void pagerUnlockIfUnused(Pager *pPager){
-  if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){
+  if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){
+    assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */
     pagerUnlockAndRollback(pPager);
   }
 }
@@ -5636,25 +5637,39 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){
 /*
 ** Release a page reference.
 **
-** If the number of references to the page drop to zero, then the
-** page is added to the LRU list.  When all references to all pages
-** are released, a rollback occurs and the lock on the database is
-** removed.
+** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be
+** used if we know that the page being released is not the last page.
+** The btree layer always holds page1 open until the end, so these first
+** to routines can be used to release any page other than BtShared.pPage1.
+**
+** Use sqlite3PagerUnrefPageOne() to release page1.  This latter routine
+** checks the total number of outstanding pages and if the number of
+** pages reaches zero it drops the database lock.
 */
 void sqlite3PagerUnrefNotNull(DbPage *pPg){
-  Pager *pPager;
+  TESTONLY( Pager *pPager = pPg->pPager; )
   assert( pPg!=0 );
-  pPager = pPg->pPager;
   if( pPg->flags & PGHDR_MMAP ){
+    assert( pPg->pgno!=1 );  /* Page1 is never memory mapped */
     pagerReleaseMapPage(pPg);
   }else{
     sqlite3PcacheRelease(pPg);
   }
-  pagerUnlockIfUnused(pPager);
+  /* Do not use this routine to release the last reference to page1 */
+  assert( sqlite3PcacheRefCount(pPager->pPCache)>0 );
 }
 void sqlite3PagerUnref(DbPage *pPg){
   if( pPg ) sqlite3PagerUnrefNotNull(pPg);
 }
+void sqlite3PagerUnrefPageOne(DbPage *pPg){
+  Pager *pPager;
+  assert( pPg!=0 );
+  assert( pPg->pgno==1 );
+  assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */
+  pPager = pPg->pPager;
+  sqlite3PcacheRelease(pPg);
+  pagerUnlockIfUnused(pPager);
+}
 
 /*
 ** This function is called at the start of every write transaction.
diff --git a/src/pager.h b/src/pager.h
index 585ef29497..126267bcc8 100644
--- a/src/pager.h
+++ b/src/pager.h
@@ -151,6 +151,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno);
 void sqlite3PagerRef(DbPage*);
 void sqlite3PagerUnref(DbPage*);
 void sqlite3PagerUnrefNotNull(DbPage*);
+void sqlite3PagerUnrefPageOne(DbPage*);
 
 /* Operations on page references. */
 int sqlite3PagerWrite(DbPage*);

From 70739addc29acef1cfd4ca49f8c3bf7148b3ef08 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 4 Sep 2017 00:19:29 +0000
Subject: [PATCH 062/270] Fix recent test cases in test/indexepxr2.test so that
 they work even without SQLITE_ENABLE_STAT4 and SQLITE_ENABLE_JSON1.

FossilOrigin-Name: 03f3cc03aaf233ad663d32a0200bfafee24a6c81f6a0ad14094ff014f880f00f
---
 manifest             | 12 +++++------
 manifest.uuid        |  2 +-
 test/indexexpr2.test | 51 ++++++++++++++++++++++----------------------
 3 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index 633c8bc712..76aa15ba3f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\snew\ssqlite3PagerUnrefPageOne()\spager\smethod\sto\sdeal\swith\sthe\sspecial\ncase\sof\sunreferencing\spage1.
-D 2017-09-01T14:50:19.167
+C Fix\srecent\stest\scases\sin\stest/indexepxr2.test\sso\sthat\sthey\swork\seven\swithout\nSQLITE_ENABLE_STAT4\sand\sSQLITE_ENABLE_JSON1.
+D 2017-09-04T00:19:29.067
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -943,7 +943,7 @@ F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
-F test/indexexpr2.test 2237f1408efa921bd66d0a09ebf0208cb0c228c1bc3b3a18e9fb8fc87d6ed90b
+F test/indexexpr2.test fdccd5c13a57af59a8e392660953dbcaacc4699c433516372cfba52994aa503a
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c4e7e175eecfd79015f4fae99618dfce6baf97c21bf3c909ea535d4e12dcaaad
-R 6f9a52521f107a3c330b7cfcb23523cf
+P b2e2100cf766da1cb499aec0ca0a9969d7af5d93312ab8cf895fbf4a6ffb9b2f
+R 433d5cb6c81e4d88ac4aa41130f6c962
 U drh
-Z f39368718f6b0d67941da00fc11a34d5
+Z bece2044ea92becf4529a710cefa9f07
diff --git a/manifest.uuid b/manifest.uuid
index 1aa72acac9..cfb4284505 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b2e2100cf766da1cb499aec0ca0a9969d7af5d93312ab8cf895fbf4a6ffb9b2f
\ No newline at end of file
+03f3cc03aaf233ad663d32a0200bfafee24a6c81f6a0ad14094ff014f880f00f
\ No newline at end of file
diff --git a/test/indexexpr2.test b/test/indexexpr2.test
index 7fa226cf8f..d8b20934e7 100644
--- a/test/indexexpr2.test
+++ b/test/indexexpr2.test
@@ -57,10 +57,7 @@ do_eqp_test 3.1.1 {
   SELECT b FROM t1 WHERE b IS NOT NULL AND a IS NULL 
   GROUP BY b COLLATE nocase
   ORDER BY b COLLATE nocase;
-} {
-  0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=? AND b>?)}
-  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
-}
+} {/USE TEMP B-TREE FOR GROUP BY/}
 
 do_execsql_test 3.2.0 {
   CREATE TABLE t2(x);
@@ -88,28 +85,30 @@ do_execsql_test 3.3.0 {
   CREATE TABLE t3(x);
 }
 
-do_eqp_test 3.3.1 {
-  SELECT json_extract(x, '$.b') FROM t2 
-  WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
-  GROUP BY json_extract(x, '$.b') COLLATE nocase
-  ORDER BY json_extract(x, '$.b') COLLATE nocase;
-} {
-  0 0 0 {SCAN TABLE t2} 
-  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
-}
-
-do_execsql_test 3.3.2 {
-  CREATE INDEX i3 ON t3(json_extract(x, '$.a'), json_extract(x, '$.b'));
-} {}
-
-do_eqp_test 3.3.3 {
-  SELECT json_extract(x, '$.b') FROM t3 
-  WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
-  GROUP BY json_extract(x, '$.b') COLLATE nocase
-  ORDER BY json_extract(x, '$.b') COLLATE nocase;
-} {
-  0 0 0 {SEARCH TABLE t3 USING INDEX i3 (<expr>=?)} 
-  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+ifcapable json1 {
+  do_eqp_test 3.3.1 {
+    SELECT json_extract(x, '$.b') FROM t2 
+    WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
+    GROUP BY json_extract(x, '$.b') COLLATE nocase
+    ORDER BY json_extract(x, '$.b') COLLATE nocase;
+  } {
+    0 0 0 {SCAN TABLE t2} 
+    0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+  }
+  
+  do_execsql_test 3.3.2 {
+    CREATE INDEX i3 ON t3(json_extract(x, '$.a'), json_extract(x, '$.b'));
+  } {}
+  
+  do_eqp_test 3.3.3 {
+    SELECT json_extract(x, '$.b') FROM t3 
+    WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL 
+    GROUP BY json_extract(x, '$.b') COLLATE nocase
+    ORDER BY json_extract(x, '$.b') COLLATE nocase;
+  } {
+    0 0 0 {SEARCH TABLE t3 USING INDEX i3 (<expr>=?)} 
+    0 0 0 {USE TEMP B-TREE FOR GROUP BY}
+  }
 }
 
 do_execsql_test 3.4.0 {

From 7e6f980bd461201928011e2f8356f86e2bbf02a9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 4 Sep 2017 00:33:04 +0000
Subject: [PATCH 063/270] Proposed fix for ticket [b899b6042f97f5]

FossilOrigin-Name: c7f9f47b239fdd99b555fb0a31eb82b22dbe6c821f1612d67a0d6a822288d9f7
---
 manifest         | 20 ++++++++++----------
 manifest.uuid    |  2 +-
 src/expr.c       | 26 +++++++++++++++++---------
 src/sqliteInt.h  |  1 +
 src/where.c      |  1 +
 test/eqp.test    |  6 +++---
 test/whereF.test | 10 ++++++++++
 7 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index 76aa15ba3f..40ea8c39a0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\srecent\stest\scases\sin\stest/indexepxr2.test\sso\sthat\sthey\swork\seven\swithout\nSQLITE_ENABLE_STAT4\sand\sSQLITE_ENABLE_JSON1.
-D 2017-09-04T00:19:29.067
+C Proposed\sfix\sfor\sticket\s[b899b6042f97f5]
+D 2017-09-04T00:33:04.196
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -409,7 +409,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 4ca86dc65f5ea478c665a5b4fe79d05f00432c9bd82237a896b45bd376bf1217
+F src/expr.c 0f611840217016cf2c5e72f2eb8e412e48511bf740ae1fd5b58dc5e409c6e738
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -462,7 +462,7 @@ F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469
 F src/sqlite.h.in 21f62793a3611b43f6fb31f0a4c8b38489a4df025416e9b7db7cc01bf5ef5aaa
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 60295f5f909e32aef1961075a8fa98df19335a4b7792b4a0b897f1d8789681c9
+F src/sqliteInt.h f9ae3609a583aa51712083e1d5817f62c7d97c0a203036a9a62c439059e8258b
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -538,7 +538,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 0aaa1b085a018c1c2e2da367b48491ce2686aea351bd17772c46b7e04eb51e3d
+F src/where.c 101f0a645c45c12141b38a61b593232555fc001bf7786dcb03eb8f313783b404
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
 F src/wherecode.c d246d19f5453d3f154ed8fcea892ce6d70ae4a5ddaebae34bd49d73f4c913bc7
 F src/whereexpr.c fe1fe600d7334e91f3d9d487021362d543fba8ab2f1be5e0d68063d619379c05
@@ -750,7 +750,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea
 F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473
 F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6
 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
-F test/eqp.test 3f9ba0b2594837c7beaa3ba824e2137cfe857308f020ec5a0c7a62b444e837b0
+F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2
 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9
 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c
 F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
@@ -1530,7 +1530,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test 97a86ecdfa4c21684fdff501dbd2cb7397689be8676d0dbad1f5a0892c6b56a3
+F test/whereF.test ec178da9a65f50d1fefeb0fd1303faa01fe74d1eec5b24db89b040b7c4faa9c7
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b2e2100cf766da1cb499aec0ca0a9969d7af5d93312ab8cf895fbf4a6ffb9b2f
-R 433d5cb6c81e4d88ac4aa41130f6c962
+P 03f3cc03aaf233ad663d32a0200bfafee24a6c81f6a0ad14094ff014f880f00f
+R 73122b1ef41dd788e5b58ec3a808a0c7
 U drh
-Z bece2044ea92becf4529a710cefa9f07
+Z eed146bdd0a46f231fdb1a0c698d494d
diff --git a/manifest.uuid b/manifest.uuid
index cfb4284505..81c3c444e8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-03f3cc03aaf233ad663d32a0200bfafee24a6c81f6a0ad14094ff014f880f00f
\ No newline at end of file
+c7f9f47b239fdd99b555fb0a31eb82b22dbe6c821f1612d67a0d6a822288d9f7
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 0e1a8781e1..0c3b608dee 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1687,6 +1687,19 @@ u32 sqlite3ExprListFlags(const ExprList *pList){
   return m;
 }
 
+/*
+** This is a SELECT-node callback for the expression walker that
+** always "fails".  By "fail" in this case, we mean set
+** pWalker->eCode to zero and abort.
+**
+** This callback is used by multiple expression walkers.
+*/
+int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){
+  UNUSED_PARAMETER(NotUsed);
+  pWalker->eCode = 0;
+  return WRC_Abort;
+}
+
 /*
 ** These routines are Walker callbacks used to check expressions to
 ** see if they are "constant" for some definition of constant.  The
@@ -1763,21 +1776,16 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
       }
       /* Fall through */
     default:
-      testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */
-      testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */
+      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail will disallow */
+      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail will disallow */
       return WRC_Continue;
   }
 }
-static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){
-  UNUSED_PARAMETER(NotUsed);
-  pWalker->eCode = 0;
-  return WRC_Abort;
-}
 static int exprIsConst(Expr *p, int initFlag, int iCur){
   Walker w;
   w.eCode = initFlag;
   w.xExprCallback = exprNodeIsConstant;
-  w.xSelectCallback = selectNodeIsConstant;
+  w.xSelectCallback = sqlite3SelectWalkFail;
 #ifdef SQLITE_DEBUG
   w.xSelectCallback2 = sqlite3SelectWalkAssert2;
 #endif
@@ -1900,7 +1908,7 @@ int sqlite3ExprContainsSubquery(Expr *p){
   Walker w;
   w.eCode = 1;
   w.xExprCallback = sqlite3ExprWalkNoop;
-  w.xSelectCallback = selectNodeIsConstant;
+  w.xSelectCallback = sqlite3SelectWalkFail;
 #ifdef SQLITE_DEBUG
   w.xSelectCallback2 = sqlite3SelectWalkAssert2;
 #endif
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 1c1879a922..4ce54cf73d 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3368,6 +3368,7 @@ int sqlite3WalkSelectExpr(Walker*, Select*);
 int sqlite3WalkSelectFrom(Walker*, Select*);
 int sqlite3ExprWalkNoop(Walker*, Expr*);
 int sqlite3SelectWalkNoop(Walker*, Select*);
+int sqlite3SelectWalkFail(Walker*, Select*);
 #ifdef SQLITE_DEBUG
 void sqlite3SelectWalkAssert2(Walker*, Select*);
 #endif
diff --git a/src/where.c b/src/where.c
index d414b936d5..7541873023 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4328,6 +4328,7 @@ static int exprIsDeterministic(Expr *p){
   memset(&w, 0, sizeof(w));
   w.eCode = 1;
   w.xExprCallback = exprNodeIsDeterministic;
+  w.xSelectCallback = sqlite3SelectWalkFail;
   sqlite3WalkExpr(&w, p);
   return w.eCode;
 }
diff --git a/test/eqp.test b/test/eqp.test
index 30fcdf287f..c955a80c21 100644
--- a/test/eqp.test
+++ b/test/eqp.test
@@ -188,24 +188,24 @@ do_eqp_test 3.1.1 {
 do_eqp_test 3.1.2 {
   SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub);
 } {
+  0 0 0 {SCAN TABLE t1}
   0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   1 0 0 {SCAN TABLE t1 AS sub}
-  0 0 0 {SCAN TABLE t1}
 }
 do_eqp_test 3.1.3 {
   SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub ORDER BY y);
 } {
+  0 0 0 {SCAN TABLE t1}
   0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   1 0 0 {SCAN TABLE t1 AS sub}
   1 0 0 {USE TEMP B-TREE FOR ORDER BY}
-  0 0 0 {SCAN TABLE t1}
 }
 do_eqp_test 3.1.4 {
   SELECT * FROM t1 WHERE (SELECT x FROM t2 ORDER BY x);
 } {
+  0 0 0 {SCAN TABLE t1}
   0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   1 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1}
-  0 0 0 {SCAN TABLE t1}
 }
 
 det 3.2.1 {
diff --git a/test/whereF.test b/test/whereF.test
index 3b938aa203..7b42e0a19f 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -176,4 +176,14 @@ do_execsql_test 5.5 {
 } {4}
 do_test 5.6 { expr [db status vmstep]<200 } 1
 
+# 2017-09-04 ticket b899b6042f97f52d
+# Segfault on correlated subquery...
+#
+ifcapable json1 {
+  do_execsql_test 6.1 {
+    CREATE TABLE t6(x);
+    SELECT * FROM t6 WHERE 1 IN (SELECT value FROM json_each(x));
+  } {}
+}
+
 finish_test

From 050dc5cff48a098e97b8528a39a51a402c1286a6 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 4 Sep 2017 18:44:54 +0000
Subject: [PATCH 064/270] Add the 'mksourceid' executable to the clean target
 for MSVC.

FossilOrigin-Name: 77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5
---
 Makefile.msc  |  2 +-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index a1937fe5fa..1289fe6d5b 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2266,7 +2266,7 @@ clean:
 	del /Q sqlite3.c sqlite3.h 2>NUL
 	del /Q opcodes.c opcodes.h 2>NUL
 	del /Q lemon.* lempar.c parse.* 2>NUL
-	del /Q mkkeywordhash.* keywordhash.h 2>NUL
+	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
 	del /Q notasharedlib.* 2>NUL
 	-rmdir /Q/S .deps 2>NUL
 	-rmdir /Q/S .libs 2>NUL
diff --git a/manifest b/manifest
index 40ea8c39a0..36df2c9a30 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Proposed\sfix\sfor\sticket\s[b899b6042f97f5]
-D 2017-09-04T00:33:04.196
+C Add\sthe\s'mksourceid'\sexecutable\sto\sthe\sclean\starget\sfor\sMSVC.
+D 2017-09-04T18:44:54.145
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
+F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 03f3cc03aaf233ad663d32a0200bfafee24a6c81f6a0ad14094ff014f880f00f
-R 73122b1ef41dd788e5b58ec3a808a0c7
-U drh
-Z eed146bdd0a46f231fdb1a0c698d494d
+P c7f9f47b239fdd99b555fb0a31eb82b22dbe6c821f1612d67a0d6a822288d9f7
+R b36b0263a36e16d360d8708a28fbd7f3
+U mistachkin
+Z c44112597706424847ee8ef603a5780f
diff --git a/manifest.uuid b/manifest.uuid
index 81c3c444e8..1f59a93e08 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c7f9f47b239fdd99b555fb0a31eb82b22dbe6c821f1612d67a0d6a822288d9f7
\ No newline at end of file
+77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5
\ No newline at end of file

From 88a79730572ab7566c73ae102f20c85bb2162423 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 4 Sep 2017 19:31:54 +0000
Subject: [PATCH 065/270] Use the SQLITE_CORRUPT_BKPT return code in a couple
 more places.

FossilOrigin-Name: 72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 36df2c9a30..606b100619 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s'mksourceid'\sexecutable\sto\sthe\sclean\starget\sfor\sMSVC.
-D 2017-09-04T18:44:54.145
+C Use\sthe\sSQLITE_CORRUPT_BKPT\sreturn\scode\sin\sa\scouple\smore\splaces.
+D 2017-09-04T19:31:54.200
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -399,7 +399,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c f1ac9509f4595d034fe432982c39d51ac2bfc9bd83136d16255aede35c526b45
+F src/btree.c 1c2b2f1714c411d7a9bc52c90d9dd7eab261261d5691ac0f67e1ced92419799c
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
@@ -1651,7 +1651,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c7f9f47b239fdd99b555fb0a31eb82b22dbe6c821f1612d67a0d6a822288d9f7
-R b36b0263a36e16d360d8708a28fbd7f3
+P 77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5
+R 4f0c4978d96d81c3edc4766e1765023c
 U mistachkin
-Z c44112597706424847ee8ef603a5780f
+Z 249910ae9570afe838571fb260f0586c
diff --git a/manifest.uuid b/manifest.uuid
index 1f59a93e08..b52e0e026f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5
\ No newline at end of file
+72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 3ca0037e9d..a1b125dda8 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -775,7 +775,7 @@ static int btreeMoveto(
     if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT;
     sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey);
     if( pIdxKey->nField==0 ){
-      rc = SQLITE_CORRUPT;
+      rc = SQLITE_CORRUPT_BKPT;
       goto moveto_done;
     }
   }else{
@@ -5388,7 +5388,7 @@ int sqlite3BtreeMovetoUnpacked(
           *pRes = 0;
           rc = SQLITE_OK;
           pCur->ix = (u16)idx;
-          if( pIdxKey->errCode ) rc = SQLITE_CORRUPT;
+          if( pIdxKey->errCode ) rc = SQLITE_CORRUPT_BKPT;
           goto moveto_finish;
         }
         if( lwr>upr ) break;

From f21124f2517e5f3b97751773246e121378621f24 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 5 Sep 2017 16:24:38 +0000
Subject: [PATCH 066/270] Add experimental API sqlite3rbu_temp_size_limit().
 For limiting the amount of temporary disk space RBU uses.

FossilOrigin-Name: 7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e
---
 ext/rbu/rbutemplimit.test | 129 ++++++++++++++++++++++++++++++++++++++
 ext/rbu/sqlite3rbu.c      |  68 ++++++++++++++++++--
 ext/rbu/sqlite3rbu.h      |  22 +++++++
 ext/rbu/test_rbu.c        |  38 ++++++++---
 manifest                  |  19 +++---
 manifest.uuid             |   2 +-
 6 files changed, 253 insertions(+), 25 deletions(-)
 create mode 100644 ext/rbu/rbutemplimit.test

diff --git a/ext/rbu/rbutemplimit.test b/ext/rbu/rbutemplimit.test
new file mode 100644
index 0000000000..274f870b73
--- /dev/null
+++ b/ext/rbu/rbutemplimit.test
@@ -0,0 +1,129 @@
+# 2014 August 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbutemplimit
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+proc setup_databases {} {
+  forcedelete test.db2
+  forcedelete test.db
+  sqlite3 db test.db
+  execsql {
+    -- Create target database schema.
+    --
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100));
+    CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100));
+    CREATE INDEX i1b ON t1(b);
+    CREATE INDEX i1c ON t1(c);
+    CREATE INDEX i2b ON t2(b);
+    CREATE INDEX i2c ON t2(c);
+  
+    -- Create a large RBU database.
+    --
+    ATTACH 'test.db2' AS rbu;
+    CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10000
+    )
+    INSERT INTO data_t1 SELECT i, randomblob(100), randomblob(100), 0 FROM s;
+    CREATE TABLE rbu.data_t2(a, b, c, rbu_control);
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<15000
+    )
+    INSERT INTO data_t2 SELECT i, randomblob(100), randomblob(100), 0 FROM s;
+  }
+  db close
+}
+
+proc run_rbu_cachesize {target rbu cachesize temp_limit} {
+  sqlite3rbu rbu $target $rbu
+  rbu temp_size_limit $temp_limit
+  sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize"
+  while 1 {
+    set rc [rbu step]
+    set ::A([rbu temp_size]) 1
+    if {$rc!="SQLITE_OK"} break
+  }
+  list [catch {rbu close} msg] $msg
+}
+
+proc step_rbu_cachesize {target rbu stepsize cachesize temp_limit} {
+  set res ""
+  while 1 {
+    sqlite3rbu rbu $target $rbu
+    rbu temp_size_limit $temp_limit
+    sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize"
+    for {set i 0} {$i < $stepsize} {incr i} {
+      set rc [rbu step]
+      set ::A([rbu temp_size]) 1
+      if {$rc!="SQLITE_OK"} break
+    }
+    set res [list [catch {rbu close} msg] $msg]
+    if {$res != "0 SQLITE_OK"} break
+  }
+  set res
+}
+
+do_test 1.1.0 { setup_databases } {}
+
+do_test 1.1.1 {
+  unset -nocomplain ::A
+  run_rbu_cachesize test.db test.db2 10 0
+} {0 SQLITE_DONE}
+
+do_test 1.1.2 { llength [array names ::A] } 3
+
+do_test 1.1.3 { 
+  foreach {a0 a1 a2} [lsort -integer [array names ::A]] {}
+  list [expr $a0==0]                         \
+       [expr $a1>1048576] [expr $a1<1200000] \
+       [expr $a2>1500000] [expr $a2<1700000]
+} {1 1 1 1 1}
+
+do_test 1.2.1 {
+  setup_databases
+  run_rbu_cachesize test.db test.db2 10 1000000
+} {1 SQLITE_FULL}
+do_test 1.2.2 { info commands rbu } {}
+
+do_test 1.3.1 {
+  setup_databases
+  run_rbu_cachesize test.db test.db2 10 1300000
+} {1 SQLITE_FULL}
+do_test 1.3.2 { info commands rbu } {}
+
+do_test 1.4.1 {
+  setup_databases
+  run_rbu_cachesize test.db test.db2 10 1800000
+} {0 SQLITE_DONE}
+do_test 1.4.2 { info commands rbu } {}
+
+do_test 1.5.1 {
+  setup_databases
+  unset -nocomplain ::A
+  step_rbu_cachesize test.db test.db2 1000 10 2400000
+} {0 SQLITE_DONE}
+do_test 1.5.2 { info commands rbu } {}
+
+do_test 1.6.1 {
+  setup_databases
+  unset -nocomplain ::A
+  step_rbu_cachesize test.db test.db2 1000 10 1400000
+} {1 SQLITE_FULL}
+do_test 1.6.2 { info commands rbu } {}
+
+finish_test
+
diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c
index 033127853b..fbcfab9104 100644
--- a/ext/rbu/sqlite3rbu.c
+++ b/ext/rbu/sqlite3rbu.c
@@ -371,6 +371,8 @@ struct sqlite3rbu {
   int pgsz;
   u8 *aBuf;
   i64 iWalCksum;
+  i64 szTemp;                     /* Current size of all temp files in use */
+  i64 szTempLimit;                /* Total size limit for temp files */
 
   /* Used in RBU vacuum mode only */
   int nRbu;                       /* Number of RBU VFS in the stack */
@@ -379,23 +381,33 @@ struct sqlite3rbu {
 
 /*
 ** An rbu VFS is implemented using an instance of this structure.
+**
+** Variable pRbu is only non-NULL for automatically created RBU VFS objects.
+** It is NULL for RBU VFS objects created explicitly using
+** sqlite3rbu_create_vfs(). It is used to track the total amount of temp
+** space used by the RBU handle.
 */
 struct rbu_vfs {
   sqlite3_vfs base;               /* rbu VFS shim methods */
   sqlite3_vfs *pRealVfs;          /* Underlying VFS */
   sqlite3_mutex *mutex;           /* Mutex to protect pMain */
+  sqlite3rbu *pRbu;               /* Owner RBU object */
   rbu_file *pMain;                /* Linked list of main db files */
 };
 
 /*
 ** Each file opened by an rbu VFS is represented by an instance of
 ** the following structure.
+**
+** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable
+** "sz" is set to the current size of the database file.
 */
 struct rbu_file {
   sqlite3_file base;              /* sqlite3_file methods */
   sqlite3_file *pReal;            /* Underlying file handle */
   rbu_vfs *pRbuVfs;               /* Pointer to the rbu_vfs object */
   sqlite3rbu *pRbu;               /* Pointer to rbu object (rbu target only) */
+  i64 sz;                         /* Size of file in bytes (temp only) */
 
   int openFlags;                  /* Flags this file was opened with */
   u32 iCookie;                    /* Cookie value for main db files */
@@ -3409,6 +3421,7 @@ static void rbuCreateVfs(sqlite3rbu *p){
     sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
     assert( pVfs );
     p->zVfsName = pVfs->zName;
+    ((rbu_vfs*)pVfs)->pRbu = p;
   }
 }
 
@@ -3781,6 +3794,7 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){
     /* Close the open database handle and VFS object. */
     sqlite3_close(p->dbRbu);
     sqlite3_close(p->dbMain);
+    assert( p->szTemp==0 );
     rbuDeleteVfs(p);
     sqlite3_free(p->aBuf);
     sqlite3_free(p->aFrame);
@@ -3968,6 +3982,7 @@ int sqlite3rbu_savestate(sqlite3rbu *p){
 */
 
 static void rbuUnlockShm(rbu_file *p){
+  assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
   if( p->pRbu ){
     int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
     int i;
@@ -3980,6 +3995,18 @@ static void rbuUnlockShm(rbu_file *p){
   }
 }
 
+/*
+*/
+static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){
+  sqlite3rbu *pRbu = pFd->pRbu;
+  i64 nDiff = nNew - pFd->sz;
+  pRbu->szTemp += nDiff;
+  pFd->sz = nNew;
+  assert( pRbu->szTemp>=0 );
+  if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
+  return SQLITE_OK;
+}
+
 /*
 ** Close an rbu file.
 */
@@ -4005,6 +4032,9 @@ static int rbuVfsClose(sqlite3_file *pFile){
     rbuUnlockShm(p);
     p->pReal->pMethods->xShmUnmap(p->pReal, 0);
   }
+  else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
+    rbuUpdateTempSize(p, 0);
+  }
 
   /* Close the underlying file handle */
   rc = p->pReal->pMethods->xClose(p->pReal);
@@ -4122,11 +4152,19 @@ static int rbuVfsWrite(
     assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
     rc = rbuCaptureDbWrite(p->pRbu, iOfst);
   }else{
-    if( pRbu && pRbu->eStage==RBU_STAGE_OAL 
-     && (p->openFlags & SQLITE_OPEN_WAL) 
-     && iOfst>=pRbu->iOalSz
-    ){
-      pRbu->iOalSz = iAmt + iOfst;
+    if( pRbu ){
+      if( pRbu->eStage==RBU_STAGE_OAL 
+       && (p->openFlags & SQLITE_OPEN_WAL) 
+       && iOfst>=pRbu->iOalSz
+      ){
+        pRbu->iOalSz = iAmt + iOfst;
+      }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){
+        i64 szNew = iAmt+iOfst;
+        if( szNew>p->sz ){
+          rc = rbuUpdateTempSize(p, szNew);
+          if( rc!=SQLITE_OK ) return rc;
+        }
+      }
     }
     rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
     if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
@@ -4145,6 +4183,10 @@ static int rbuVfsWrite(
 */
 static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
   rbu_file *p = (rbu_file*)pFile;
+  if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
+    int rc = rbuUpdateTempSize(p, size);
+    if( rc!=SQLITE_OK ) return rc;
+  }
   return p->pReal->pMethods->xTruncate(p->pReal, size);
 }
 
@@ -4534,6 +4576,8 @@ static int rbuVfsOpen(
         pDb->pWalFd = pFd;
       }
     }
+  }else{
+    pFd->pRbu = pRbuVfs->pRbu;
   }
 
   if( oflags & SQLITE_OPEN_MAIN_DB 
@@ -4801,6 +4845,20 @@ int sqlite3rbu_create_vfs(const char *zName, const char *zParent){
   return rc;
 }
 
+/*
+** Configure the aggregate temp file size limit for this RBU handle.
+*/
+sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){
+  if( n>=0 ){
+    pRbu->szTempLimit = n;
+  }
+  return pRbu->szTempLimit;
+}
+
+sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){
+  return pRbu->szTemp;
+}
+
 
 /**************************************************************************/
 
diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h
index 2f038fd8fd..1acbcca469 100644
--- a/ext/rbu/sqlite3rbu.h
+++ b/ext/rbu/sqlite3rbu.h
@@ -352,6 +352,28 @@ SQLITE_API sqlite3rbu *sqlite3rbu_vacuum(
   const char *zState
 );
 
+/*
+** Configure a limit for the amount of temp space that may be used by
+** the RBU handle passed as the first argument. The new limit is specified
+** in bytes by the second parameter. If it is positive, the limit is updated.
+** If the second parameter to this function is passed zero, then the limit
+** is removed entirely. If the second parameter is negative, the limit is
+** not modified (this is useful for querying the current limit).
+**
+** In all cases the returned value is the current limit in bytes (zero 
+** indicates unlimited).
+**
+** If the temp space limit is exceeded during operation, an SQLITE_FULL
+** error is returned.
+*/
+SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64);
+
+/*
+** Return the current amount of temp file space, in bytes, currently used by 
+** the RBU handle passed as the only argument.
+*/
+SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*);
+
 /*
 ** Internally, each RBU connection uses a separate SQLite database 
 ** connection to access the target and rbu update databases. This
diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c
index fba90dcdc4..631bff2a76 100644
--- a/ext/rbu/test_rbu.c
+++ b/ext/rbu/test_rbu.c
@@ -69,16 +69,18 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
     int nArg;
     const char *zUsage;
   } aCmd[] = {
-    {"step", 2, ""},              /* 0 */
-    {"close", 2, ""},             /* 1 */
-    {"create_rbu_delta", 2, ""},  /* 2 */
-    {"savestate", 2, ""},         /* 3 */
-    {"dbMain_eval", 3, "SQL"},    /* 4 */
-    {"bp_progress", 2, ""},       /* 5 */
-    {"db", 3, "RBU"},             /* 6 */
-    {"state", 2, ""},             /* 7 */
-    {"progress", 2, ""},          /* 8 */
-    {"close_no_error", 2, ""},    /* 9 */
+    {"step", 2, ""},                 /* 0 */
+    {"close", 2, ""},                /* 1 */
+    {"create_rbu_delta", 2, ""},     /* 2 */
+    {"savestate", 2, ""},            /* 3 */
+    {"dbMain_eval", 3, "SQL"},       /* 4 */
+    {"bp_progress", 2, ""},          /* 5 */
+    {"db", 3, "RBU"},                /* 6 */
+    {"state", 2, ""},                /* 7 */
+    {"progress", 2, ""},             /* 8 */
+    {"close_no_error", 2, ""},       /* 9 */
+    {"temp_size_limit", 3, "LIMIT"}, /* 10 */
+    {"temp_size", 2, ""},            /* 11 */
     {0,0,0}
   };
   int iCmd;
@@ -193,6 +195,22 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
       Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep));
       break;
     }
+                           
+    case 10: /* temp_size_limit */ {
+      sqlite3_int64 nLimit;
+      if( Tcl_GetWideIntFromObj(interp, objv[2], &nLimit) ){
+        ret = TCL_ERROR;
+      }else{
+        nLimit = sqlite3rbu_temp_size_limit(pRbu, nLimit);
+        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nLimit));
+      }
+      break;
+    }
+    case 11: /* temp_size */ {
+      sqlite3_int64 sz = sqlite3rbu_temp_size(pRbu);
+      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(sz));
+      break;
+    }
 
     default: /* seems unlikely */
       assert( !"cannot happen" );
diff --git a/manifest b/manifest
index 606b100619..592f8659d4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sthe\sSQLITE_CORRUPT_BKPT\sreturn\scode\sin\sa\scouple\smore\splaces.
-D 2017-09-04T19:31:54.200
+C Add\sexperimental\sAPI\ssqlite3rbu_temp_size_limit().\sFor\slimiting\sthe\samount\sof\ntemporary\sdisk\sspace\sRBU\suses.
+D 2017-09-05T16:24:38.115
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -318,11 +318,12 @@ F ext/rbu/rbufts.test a2bbd202c9321fba15fb4a62a90add7d70e07bd8404e1e598135adbfff
 F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf
 F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
+F ext/rbu/rbutemplimit.test cd553a9288d515d0b5f87d277e76fd18c4aa740b761e7880fab11ce986ea18d1
 F ext/rbu/rbuvacuum.test ff357e9b556ca7ad4673da0ff7f244def919ff858e0f9f350d3e30fdd83a62a8
 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa
-F ext/rbu/sqlite3rbu.c 920941a6ff7dbbea0970717c43662878fda5c37e43752de329f0fdd76680ab75
-F ext/rbu/sqlite3rbu.h 82c102e5ae41025e3b245d3d5944315f82811da85e2cd363a75caa97cbd0cd3e
-F ext/rbu/test_rbu.c ec18cfc69a104309df23c359e3c80306c9a6bdd1d2c53c8b70ae158e9832dcd6
+F ext/rbu/sqlite3rbu.c a1a303de8b90f987ef63bf9cef57f5d7dd7983a9e8aed3775a759d87ad57075d
+F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
+F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cf84d52958a7ec6a506f1711e119db847ed6bb5dedde78a58e97503287afcda1
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1651,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5
-R 4f0c4978d96d81c3edc4766e1765023c
-U mistachkin
-Z 249910ae9570afe838571fb260f0586c
+P 72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636
+R 6f77c70118414e1c190062cb8a829104
+U dan
+Z c201de8958e66e7927017757d360b434
diff --git a/manifest.uuid b/manifest.uuid
index b52e0e026f..5ee97ca57f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636
\ No newline at end of file
+7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e
\ No newline at end of file

From 1273f8dafe1c59e4b2650789df268b3bfe582888 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 5 Sep 2017 17:23:39 +0000
Subject: [PATCH 067/270] Fix a comment describing the xRowid method of the
 "series" virtual table. No code changes.

FossilOrigin-Name: 868cd1960015dc1bf003f911dcc417a94540d1948617ea1838a48b5cff59afda
---
 ext/misc/series.c |  5 +++--
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/ext/misc/series.c b/ext/misc/series.c
index 684995f3b6..1cd45fe869 100644
--- a/ext/misc/series.c
+++ b/ext/misc/series.c
@@ -195,8 +195,9 @@ static int seriesColumn(
 }
 
 /*
-** Return the rowid for the current row.  In this implementation, the
-** rowid is the same as the output value.
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
 */
 static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
   series_cursor *pCur = (series_cursor*)cur;
diff --git a/manifest b/manifest
index 592f8659d4..5064600219 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sexperimental\sAPI\ssqlite3rbu_temp_size_limit().\sFor\slimiting\sthe\samount\sof\ntemporary\sdisk\sspace\sRBU\suses.
-D 2017-09-05T16:24:38.115
+C Fix\sa\scomment\sdescribing\sthe\sxRowid\smethod\sof\sthe\s"series"\svirtual\stable.\sNo\ncode\schanges.
+D 2017-09-05T17:23:39.813
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -276,7 +276,7 @@ F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
 F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb
-F ext/misc/series.c b0f5f346aca9b7ff7caaf0da2efb4ad462441abd4dcd92a460cb573b3ea2370b
+F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681da564
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636
-R 6f77c70118414e1c190062cb8a829104
+P 7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e
+R ba2b0e2cf633a70b38f76579945773c9
 U dan
-Z c201de8958e66e7927017757d360b434
+Z 2dd0f7547357db044b1737b583f40176
diff --git a/manifest.uuid b/manifest.uuid
index 5ee97ca57f..2e00ed73fd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e
\ No newline at end of file
+868cd1960015dc1bf003f911dcc417a94540d1948617ea1838a48b5cff59afda
\ No newline at end of file

From 3841372350083d86b37c1a2b636ad0e48a22dd08 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 5 Sep 2017 20:16:19 +0000
Subject: [PATCH 068/270] Update "PRAGMA integrity_check" to detect
 inconsistencies between a single record's header and body.

FossilOrigin-Name: 9e393a0edf1a578ddecc16a72fe3c54b75c2a650d507bcbc37c9724fbede1494
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/pragma.c     |  1 +
 test/pragma.test | 21 +++++++++++++++++++++
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 5064600219..0078fd0802 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scomment\sdescribing\sthe\sxRowid\smethod\sof\sthe\s"series"\svirtual\stable.\sNo\ncode\schanges.
-D 2017-09-05T17:23:39.813
+C Update\s"PRAGMA\sintegrity_check"\sto\sdetect\sinconsistencies\sbetween\sa\ssingle\nrecord's\sheader\sand\sbody.
+D 2017-09-05T20:16:19.056
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -450,7 +450,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
-F src/pragma.c a4e5028dfc8af4c5c347cd0e91bd2f0c0f81fcd9b2c6e0acf8da7da51df7f1fe
+F src/pragma.c faf5f9ec553b71035c63e08d31651bbd28bec826d546b32c8e9c295855ca3605
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -1095,7 +1095,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
 F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531
-F test/pragma.test f274259d6393b6681eb433beb8dd39a26ec06a4431052a4880b43b84912a3f58
+F test/pragma.test faa8b64c4be28b67b3a0d3e9c977e2feff7dd2a4b08f2fd062a95d30f8c8fd1c
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e
-R ba2b0e2cf633a70b38f76579945773c9
+P 868cd1960015dc1bf003f911dcc417a94540d1948617ea1838a48b5cff59afda
+R 83e466a9200a37ab621e4f303813e8d5
 U dan
-Z 2dd0f7547357db044b1737b583f40176
+Z 1263b9376b837ff6f4f4c7ebd74970c0
diff --git a/manifest.uuid b/manifest.uuid
index 2e00ed73fd..fd33be3a9a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-868cd1960015dc1bf003f911dcc417a94540d1948617ea1838a48b5cff59afda
\ No newline at end of file
+9e393a0edf1a578ddecc16a72fe3c54b75c2a650d507bcbc37c9724fbede1494
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index 2619f2944b..fe1328c5ce 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1583,6 +1583,7 @@ void sqlite3Pragma(
           sqlite3VdbeJumpHere(v, jmp2);
         }
         /* Verify CHECK constraints */
+        sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3);
         if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){
           ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0);
           if( db->mallocFailed==0 ){
diff --git a/test/pragma.test b/test/pragma.test
index 27400917d8..cebbf9ddc0 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -1933,5 +1933,26 @@ do_test 23.5 {
   }
 } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE}
 
+reset_db
+do_execsql_test 24.0 {
+  PRAGMA page_size = 1024;
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX i1 ON t1(b);
+  INSERT INTO t1 VALUES('a', 'b', 'c');
+  PRAGMA integrity_check;
+} {ok}
+
+set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}]
+db close
+hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263
+
+sqlite3 db test.db
+do_catchsql_test 24.1 {
+  SELECT * FROM t1;
+} {1 {database disk image is malformed}}
+do_catchsql_test 24.2 {
+  PRAGMA integrity_check;
+} {1 {database disk image is malformed}}
+
 database_never_corrupt
 finish_test

From 05f1ba0ef83eafc2164898a2ce9f4df691a1ab1a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 7 Sep 2017 09:56:37 +0000
Subject: [PATCH 069/270] Prevent a possible crash when trying to recover using
 a carefully corrupted \ and truncated rollback journal.  (Test case in TH3)

FossilOrigin-Name: 02828d717e2d97b1f59e9279cea9c06eed4accd4e262606bd90d060449c5a1a3
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/pager.c   |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 0078fd0802..c758b51493 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\s"PRAGMA\sintegrity_check"\sto\sdetect\sinconsistencies\sbetween\sa\ssingle\nrecord's\sheader\sand\sbody.
-D 2017-09-05T20:16:19.056
+C Prevent\sa\spossible\scrash\swhen\strying\sto\srecover\susing\sa\scarefully\scorrupted\s\\\nand\struncated\srollback\sjournal.\s\s(Test\scase\sin\sTH3)
+D 2017-09-07T09:56:37.081
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -444,7 +444,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 3936b32a0a45f1b80462b38ba1ed0556e62843554a089ab9c4be1a35c97f23bc
+F src/pager.c 967168bba88d2dc790ed9618bd4ba7bfe475b67b521ef6da305a6425c592928f
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 868cd1960015dc1bf003f911dcc417a94540d1948617ea1838a48b5cff59afda
-R 83e466a9200a37ab621e4f303813e8d5
-U dan
-Z 1263b9376b837ff6f4f4c7ebd74970c0
+P 9e393a0edf1a578ddecc16a72fe3c54b75c2a650d507bcbc37c9724fbede1494
+R f5c3e71f60efc1950a71b5fb4b758fc2
+U drh
+Z 445858944243f3b3fef9b85f002c6409
diff --git a/manifest.uuid b/manifest.uuid
index fd33be3a9a..f41cbd8d46 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9e393a0edf1a578ddecc16a72fe3c54b75c2a650d507bcbc37c9724fbede1494
\ No newline at end of file
+02828d717e2d97b1f59e9279cea9c06eed4accd4e262606bd90d060449c5a1a3
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index b3ec03bd95..4f3f75b6e7 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -1315,6 +1315,7 @@ static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){
    || szJ<16
    || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len))
    || len>=nMaster 
+   || len>szJ-16
    || len==0 
    || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum))
    || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8))

From 7834551c1e77b83839459505e97e3b32bcffd740 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 8 Sep 2017 17:48:00 +0000
Subject: [PATCH 070/270] Have the header comment for sqlite3Checkpoint()
 mention TRUNCATE along with the other three checkpoint types. No changes to
 code.

FossilOrigin-Name: e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/main.c    |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index c758b51493..6cb2f61ec6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Prevent\sa\spossible\scrash\swhen\strying\sto\srecover\susing\sa\scarefully\scorrupted\s\\\nand\struncated\srollback\sjournal.\s\s(Test\scase\sin\sTH3)
-D 2017-09-07T09:56:37.081
+C Have\sthe\sheader\scomment\sfor\ssqlite3Checkpoint()\smention\sTRUNCATE\salong\swith\nthe\sother\sthree\scheckpoint\stypes.\sNo\schanges\sto\scode.
+D 2017-09-08T17:48:00.753
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -422,7 +422,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 48a641949ce76c857b8771230c4304501287285714ea796f803b4178629ed560
+F src/main.c 34a58563358fe40979186124d1a3614b9a00c833124d7ebfa8e5d604ed1d2521
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9e393a0edf1a578ddecc16a72fe3c54b75c2a650d507bcbc37c9724fbede1494
-R f5c3e71f60efc1950a71b5fb4b758fc2
-U drh
-Z 445858944243f3b3fef9b85f002c6409
+P 02828d717e2d97b1f59e9279cea9c06eed4accd4e262606bd90d060449c5a1a3
+R a16bad8b23f653646029e20ce84ed88a
+U dan
+Z 83e6954e72708437b1490d8e46bc78b4
diff --git a/manifest.uuid b/manifest.uuid
index f41cbd8d46..e1af05939a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02828d717e2d97b1f59e9279cea9c06eed4accd4e262606bd90d060449c5a1a3
\ No newline at end of file
+e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 168ce851f6..3942ed9465 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2173,7 +2173,8 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
 ** checkpointed. If an error is encountered it is returned immediately -
 ** no attempt is made to checkpoint any remaining databases.
 **
-** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.
+** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART
+** or TRUNCATE.
 */
 int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){
   int rc = SQLITE_OK;             /* Return code */

From 2158a0c7acd8f67cf747b50b59609f82934e7402 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 9 Sep 2017 00:51:36 +0000
Subject: [PATCH 071/270] Fix harmless compiler warnings seen with MSVC.

FossilOrigin-Name: faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
---
 manifest       | 16 ++++++++--------
 manifest.uuid  |  2 +-
 src/shell.c    |  2 +-
 src/shell.c.in |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 6cb2f61ec6..4b71f03671 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sheader\scomment\sfor\ssqlite3Checkpoint()\smention\sTRUNCATE\salong\swith\nthe\sother\sthree\scheckpoint\stypes.\sNo\schanges\sto\scode.
-D 2017-09-08T17:48:00.753
+C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC.
+D 2017-09-09T00:51:36.496
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -458,8 +458,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
-F src/shell.c 319082accd4b719a332f4cf92347546e310327b4b4d50c42de129787b5ed2946
-F src/shell.c.in af3fb9eabdc0a95beace2f760597d213be0988c974eca116208eb220cd24469c
+F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
+F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in 21f62793a3611b43f6fb31f0a4c8b38489a4df025416e9b7db7cc01bf5ef5aaa
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02828d717e2d97b1f59e9279cea9c06eed4accd4e262606bd90d060449c5a1a3
-R a16bad8b23f653646029e20ce84ed88a
-U dan
-Z 83e6954e72708437b1490d8e46bc78b4
+P e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
+R 14621bd82abfa0f650a4c6242c2108d7
+U mistachkin
+Z 11f83c8a0999c8fee516d7917728daf2
diff --git a/manifest.uuid b/manifest.uuid
index e1af05939a..ce2953ddb7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
\ No newline at end of file
+faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index a87aab2bd8..6a19e3610a 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -3074,7 +3074,7 @@ static void createSelftestTable(ShellState *p){
 */
 static void set_table_name(ShellState *p, const char *zName){
   int i, n;
-  int cQuote;
+  char cQuote;
   char *z;
 
   if( p->zDestTable ){
diff --git a/src/shell.c.in b/src/shell.c.in
index bf5e335857..82680d484c 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1714,7 +1714,7 @@ static void createSelftestTable(ShellState *p){
 */
 static void set_table_name(ShellState *p, const char *zName){
   int i, n;
-  int cQuote;
+  char cQuote;
   char *z;
 
   if( p->zDestTable ){

From d03024d81429ec770c82a01b19c181e0322be729 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 9 Sep 2017 19:41:12 +0000
Subject: [PATCH 072/270] Enhance the vtab interface to handle IS, !=, IS NOT,
 IS NULL and IS NOT NULL constraints.

FossilOrigin-Name: 34c8e952616013deb6fffec701ac5989afac9bef1bf92458a2e4ba92c7ee924f
---
 manifest             |  28 ++++---
 manifest.uuid        |   2 +-
 src/sqlite.h.in      |   5 ++
 src/test8.c          |  21 ++---
 src/test_bestindex.c |  10 +++
 src/where.c          |  53 ++++++------
 src/wherecode.c      |   2 +-
 src/whereexpr.c      | 149 ++++++++++++++++++++-------------
 test/bestindex5.test | 192 +++++++++++++++++++++++++++++++++++++++++++
 9 files changed, 359 insertions(+), 103 deletions(-)
 create mode 100644 test/bestindex5.test

diff --git a/manifest b/manifest
index 4b71f03671..3e1369face 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC.
-D 2017-09-09T00:51:36.496
+C Enhance\sthe\svtab\sinterface\sto\shandle\sIS,\s!=,\sIS\sNOT,\sIS\sNULL\sand\sIS\sNOT\sNULL\nconstraints.
+D 2017-09-09T19:41:12.986
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -460,7 +460,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
-F src/sqlite.h.in 21f62793a3611b43f6fb31f0a4c8b38489a4df025416e9b7db7cc01bf5ef5aaa
+F src/sqlite.h.in d0ab3cae93cc9819f9e7ba5c8c8e3708e657c6cdbc61ecfa7dfadd19c0308ffa
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h f9ae3609a583aa51712083e1d5817f62c7d97c0a203036a9a62c439059e8258b
@@ -475,12 +475,12 @@ F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
 F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d
 F src/test6.c e8d839fbc552ce044bec8234561a2d5b8819b48e29548ad0ba400471697946a8
 F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010
-F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86
+F src/test8.c 3f7d0cc4e12e06832ba3db4455cb16867ccadafa602eb6ff5fcf097bffce56ed
 F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5
 F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a
 F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871
 F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
-F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96
+F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
 F src/test_config.c abf6fc1fe9d041b699578c42e3db81f8831c4f5b804f1927958102ee8f2b773e
@@ -539,10 +539,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 101f0a645c45c12141b38a61b593232555fc001bf7786dcb03eb8f313783b404
+F src/where.c 7cc9692dc4f270f5a196d33d2ee1011ce6218a6061b73df592771a404ee3500c
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
-F src/wherecode.c d246d19f5453d3f154ed8fcea892ce6d70ae4a5ddaebae34bd49d73f4c913bc7
-F src/whereexpr.c fe1fe600d7334e91f3d9d487021362d543fba8ab2f1be5e0d68063d619379c05
+F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
+F src/whereexpr.c ffc3c90f68ad28c6eca1c8b05029f361bc151187be578985d992788d31f295ae
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -610,6 +610,7 @@ F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
 F test/bestindex3.test 578b6a52dab819e63f28e3640e04b32c85aed320
 F test/bestindex4.test 4cb5ff7dbaebadb87d366f51969271778423b455
+F test/bestindex5.test a7f1c32dc21d5c85afd4e9611e1160247107387b85a371fded36852c1c4959e0
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -1652,7 +1653,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
-R 14621bd82abfa0f650a4c6242c2108d7
-U mistachkin
-Z 11f83c8a0999c8fee516d7917728daf2
+P faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
+R 71eba3dbcd321cff2772c9856db2cb4e
+T *branch * vtab-extra-ops
+T *sym-vtab-extra-ops *
+T -sym-trunk *
+U dan
+Z f5cb136cb2d9007b3611d202913746bf
diff --git a/manifest.uuid b/manifest.uuid
index ce2953ddb7..e02149a44e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
\ No newline at end of file
+34c8e952616013deb6fffec701ac5989afac9bef1bf92458a2e4ba92c7ee924f
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 8b3c22c92d..e2edfe4605 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6256,6 +6256,11 @@ struct sqlite3_index_info {
 #define SQLITE_INDEX_CONSTRAINT_LIKE   65
 #define SQLITE_INDEX_CONSTRAINT_GLOB   66
 #define SQLITE_INDEX_CONSTRAINT_REGEXP 67
+#define SQLITE_INDEX_CONSTRAINT_NE        68
+#define SQLITE_INDEX_CONSTRAINT_ISNOT     69
+#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70
+#define SQLITE_INDEX_CONSTRAINT_ISNULL    71
+#define SQLITE_INDEX_CONSTRAINT_IS        72
 
 /*
 ** CAPI3REF: Register A Virtual Table Implementation
diff --git a/src/test8.c b/src/test8.c
index daab504e4e..2684f801fa 100644
--- a/src/test8.c
+++ b/src/test8.c
@@ -897,17 +897,18 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         case SQLITE_INDEX_CONSTRAINT_REGEXP:
           zOp = "regexp"; break;
       }
-      if( zOp[0]=='L' ){
-        zNew = sqlite3_mprintf(" %s %s LIKE (SELECT '%%'||?||'%%')", 
-                               zSep, zNewCol);
-      } else {
-        zNew = sqlite3_mprintf(" %s %s %s ?", zSep, zNewCol, zOp);
+      if( zOp ){
+        if( zOp[0]=='L' ){
+          zNew = sqlite3_mprintf(" %s %s LIKE (SELECT '%%'||?||'%%')", 
+              zSep, zNewCol);
+        } else {
+          zNew = sqlite3_mprintf(" %s %s %s ?", zSep, zNewCol, zOp);
+        }
+        string_concat(&zQuery, zNew, 1, &rc);
+        zSep = "AND";
+        pUsage->argvIndex = ++nArg;
+        pUsage->omit = 1;
       }
-      string_concat(&zQuery, zNew, 1, &rc);
-
-      zSep = "AND";
-      pUsage->argvIndex = ++nArg;
-      pUsage->omit = 1;
     }
   }
 
diff --git a/src/test_bestindex.c b/src/test_bestindex.c
index 94017f0349..2cd79baf2b 100644
--- a/src/test_bestindex.c
+++ b/src/test_bestindex.c
@@ -414,6 +414,16 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         zOp = "glob"; break;
       case SQLITE_INDEX_CONSTRAINT_REGEXP:
         zOp = "regexp"; break;
+      case SQLITE_INDEX_CONSTRAINT_NE:
+        zOp = "ne"; break;
+      case SQLITE_INDEX_CONSTRAINT_ISNOT:
+        zOp = "isnot"; break;
+      case SQLITE_INDEX_CONSTRAINT_ISNOTNULL:
+        zOp = "isnotnull"; break;
+      case SQLITE_INDEX_CONSTRAINT_ISNULL:
+        zOp = "isnull"; break;
+      case SQLITE_INDEX_CONSTRAINT_IS:
+        zOp = "is"; break;
     }
 
     Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("op", -1));
diff --git a/src/where.c b/src/where.c
index 7541873023..0e908e1516 100644
--- a/src/where.c
+++ b/src/where.c
@@ -868,7 +868,7 @@ static sqlite3_index_info *allocateIndexInfo(
     testcase( pTerm->eOperator & WO_ISNULL );
     testcase( pTerm->eOperator & WO_IS );
     testcase( pTerm->eOperator & WO_ALL );
-    if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
+    if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
     if( pTerm->wtFlags & TERM_VNULL ) continue;
     assert( pTerm->u.leftColumn>=(-1) );
     nTerm++;
@@ -916,7 +916,7 @@ static sqlite3_index_info *allocateIndexInfo(
                                                                    pUsage;
 
   for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
-    u8 op;
+    u16 op;
     if( pTerm->leftCursor != pSrc->iCursor ) continue;
     if( pTerm->prereqRight & mUnusable ) continue;
     assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
@@ -924,34 +924,41 @@ static sqlite3_index_info *allocateIndexInfo(
     testcase( pTerm->eOperator & WO_IS );
     testcase( pTerm->eOperator & WO_ISNULL );
     testcase( pTerm->eOperator & WO_ALL );
-    if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
+    if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
     if( pTerm->wtFlags & TERM_VNULL ) continue;
     assert( pTerm->u.leftColumn>=(-1) );
     pIdxCons[j].iColumn = pTerm->u.leftColumn;
     pIdxCons[j].iTermOffset = i;
-    op = (u8)pTerm->eOperator & WO_ALL;
+    op = pTerm->eOperator & WO_ALL;
     if( op==WO_IN ) op = WO_EQ;
     if( op==WO_MATCH ){
-      op = pTerm->eMatchOp;
-    }
-    pIdxCons[j].op = op;
-    /* The direct assignment in the previous line is possible only because
-    ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
-    ** following asserts verify this fact. */
-    assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
-    assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
-    assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
-    assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
-    assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
-    assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
-    assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
+      pIdxCons[j].op = pTerm->eMatchOp;
+    }else if( op & (WO_ISNULL|WO_IS) ){
+      if( op==WO_ISNULL ){
+        pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
+      }else{
+        pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
+      }
+    }else{
+      pIdxCons[j].op = (u8)op;
+      /* The direct assignment in the previous line is possible only because
+      ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
+      ** following asserts verify this fact. */
+      assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
+      assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
+      assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
+      assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
+      assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
+      assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
+      assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
 
-    if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
-     && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
-    ){
-      if( i<16 ) mNoOmit |= (1 << i);
-      if( op==WO_LT ) pIdxCons[j].op = WO_LE;
-      if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+      if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
+          && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
+        ){
+        if( i<16 ) mNoOmit |= (1 << i);
+        if( op==WO_LT ) pIdxCons[j].op = WO_LE;
+        if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+      }
     }
 
     j++;
diff --git a/src/wherecode.c b/src/wherecode.c
index 30ff3f25c4..6cdf7b566b 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1017,7 +1017,7 @@ static void codeDeferredSeek(
 */
 static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){
   assert( nReg>0 );
-  if( sqlite3ExprIsVector(p) ){
+  if( p && sqlite3ExprIsVector(p) ){
 #ifndef SQLITE_OMIT_SUBQUERY
     if( (p->flags & EP_xIsSelect) ){
       Vdbe *v = pParse->pVdbe;
diff --git a/src/whereexpr.c b/src/whereexpr.c
index ffd31a6ee5..45d3a7e33f 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -317,43 +317,77 @@ static int isLikeOrGlob(
 **         column OP expr
 **
 ** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a 
-** column of a virtual table.
+** column of a virtual table. If so, set *ppLeft to point to the
+** expression for "column", *ppRight to "expr" and return 1.
 **
-** If it is then return TRUE.  If not, return FALSE.
+** Also check if the expression is one of:
+**
+**         column != expr
+**         column IS NOT expr
+**         column IS NOT NULL
+**
+** where "column" is a column of a virtual table. If so, set *ppLeft
+** to point to "column", *ppRight to "expr" and return 1. Or, if "expr"
+** is also a column of a virtual table, return 2.
+**
+** If the expression matches none of the patterns above, return 0.
 */
 static int isMatchOfColumn(
   Expr *pExpr,                    /* Test this expression */
-  unsigned char *peOp2            /* OUT: 0 for MATCH, or else an op2 value */
+  unsigned char *peOp2,           /* OUT: 0 for MATCH, or else an op2 value */
+  Expr **ppLeft,                  /* Column expression to left of MATCH/op2 */
+  Expr **ppRight                  /* Expression to left of MATCH/op2 */
 ){
-  static const struct Op2 {
-    const char *zOp;
-    unsigned char eOp2;
-  } aOp[] = {
-    { "match",  SQLITE_INDEX_CONSTRAINT_MATCH },
-    { "glob",   SQLITE_INDEX_CONSTRAINT_GLOB },
-    { "like",   SQLITE_INDEX_CONSTRAINT_LIKE },
-    { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP }
-  };
-  ExprList *pList;
-  Expr *pCol;                     /* Column reference */
-  int i;
+  if( pExpr->op==TK_FUNCTION ){
+    static const struct Op2 {
+      const char *zOp;
+      unsigned char eOp2;
+    } aOp[] = {
+      { "match",  SQLITE_INDEX_CONSTRAINT_MATCH },
+      { "glob",   SQLITE_INDEX_CONSTRAINT_GLOB },
+      { "like",   SQLITE_INDEX_CONSTRAINT_LIKE },
+      { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP }
+    };
+    ExprList *pList;
+    Expr *pCol;                     /* Column reference */
+    int i;
 
-  if( pExpr->op!=TK_FUNCTION ){
-    return 0;
-  }
-  pList = pExpr->x.pList;
-  if( pList==0 || pList->nExpr!=2 ){
-    return 0;
-  }
-  pCol = pList->a[1].pExpr;
-  if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){
-    return 0;
-  }
-  for(i=0; i<ArraySize(aOp); i++){
-    if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){
-      *peOp2 = aOp[i].eOp2;
-      return 1;
+    if( pExpr->op!=TK_FUNCTION ){
+      return 0;
     }
+    pList = pExpr->x.pList;
+    if( pList==0 || pList->nExpr!=2 ){
+      return 0;
+    }
+    pCol = pList->a[1].pExpr;
+    if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){
+      return 0;
+    }
+    for(i=0; i<ArraySize(aOp); i++){
+      if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){
+        *peOp2 = aOp[i].eOp2;
+        *ppRight = pList->a[0].pExpr;
+        *ppLeft = pCol;
+        return 1;
+      }
+    }
+  }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){
+    int res = 0;
+    Expr *pLeft = pExpr->pLeft;
+    Expr *pRight = pExpr->pRight;
+    if( pLeft->op==TK_COLUMN && IsVirtual(pLeft->pTab) ){
+      res++;
+    }
+    if( pRight && pRight->op==TK_COLUMN && IsVirtual(pRight->pTab) ){
+      res++;
+      SWAP(Expr*, pLeft, pRight);
+    }
+    *ppLeft = pLeft;
+    *ppRight = pRight;
+    if( pExpr->op==TK_NE ) *peOp2 = SQLITE_INDEX_CONSTRAINT_NE;
+    if( pExpr->op==TK_ISNOT ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOT;
+    if( pExpr->op==TK_NOTNULL ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOTNULL;
+    return res;
   }
   return 0;
 }
@@ -1192,35 +1226,38 @@ static void exprAnalyze(
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
-  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
-    int idxNew;
+  if( pWC->op==TK_AND ){
     Expr *pRight, *pLeft;
-    WhereTerm *pNewTerm;
-    Bitmask prereqColumn, prereqExpr;
+    int i;
+    int res = isMatchOfColumn(pExpr, &eOp2, &pLeft, &pRight);
+    for(i=0; i<res; i++){
+      int idxNew;
+      WhereTerm *pNewTerm;
+      Bitmask prereqColumn, prereqExpr;
 
-    pRight = pExpr->x.pList->a[0].pExpr;
-    pLeft = pExpr->x.pList->a[1].pExpr;
-    prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight);
-    prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft);
-    if( (prereqExpr & prereqColumn)==0 ){
-      Expr *pNewExpr;
-      pNewExpr = sqlite3PExpr(pParse, TK_MATCH, 
-                              0, sqlite3ExprDup(db, pRight, 0));
-      if( ExprHasProperty(pExpr, EP_FromJoin) && pNewExpr ){
-        ExprSetProperty(pNewExpr, EP_FromJoin);
+      prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight);
+      prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft);
+      if( (prereqExpr & prereqColumn)==0 ){
+        Expr *pNewExpr;
+        pNewExpr = sqlite3PExpr(pParse, TK_MATCH, 
+            0, sqlite3ExprDup(db, pRight, 0));
+        if( ExprHasProperty(pExpr, EP_FromJoin) && pNewExpr ){
+          ExprSetProperty(pNewExpr, EP_FromJoin);
+        }
+        idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
+        testcase( idxNew==0 );
+        pNewTerm = &pWC->a[idxNew];
+        pNewTerm->prereqRight = prereqExpr;
+        pNewTerm->leftCursor = pLeft->iTable;
+        pNewTerm->u.leftColumn = pLeft->iColumn;
+        pNewTerm->eOperator = WO_MATCH;
+        pNewTerm->eMatchOp = eOp2;
+        markTermAsChild(pWC, idxNew, idxTerm);
+        pTerm = &pWC->a[idxTerm];
+        pTerm->wtFlags |= TERM_COPIED;
+        pNewTerm->prereqAll = pTerm->prereqAll;
       }
-      idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
-      testcase( idxNew==0 );
-      pNewTerm = &pWC->a[idxNew];
-      pNewTerm->prereqRight = prereqExpr;
-      pNewTerm->leftCursor = pLeft->iTable;
-      pNewTerm->u.leftColumn = pLeft->iColumn;
-      pNewTerm->eOperator = WO_MATCH;
-      pNewTerm->eMatchOp = eOp2;
-      markTermAsChild(pWC, idxNew, idxTerm);
-      pTerm = &pWC->a[idxTerm];
-      pTerm->wtFlags |= TERM_COPIED;
-      pNewTerm->prereqAll = pTerm->prereqAll;
+      SWAP(Expr*, pLeft, pRight);
     }
   }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
diff --git a/test/bestindex5.test b/test/bestindex5.test
new file mode 100644
index 0000000000..237041f56f
--- /dev/null
+++ b/test/bestindex5.test
@@ -0,0 +1,192 @@
+# 2017 September 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test the virtual table interface. In particular the xBestIndex
+# method.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex4
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#  
+proc vtab_cmd {method args} {
+
+  set binops(ne)    !=
+  set binops(eq)    =
+  set binops(isnot) "IS NOT"
+  set binops(is)    "IS"
+
+  set unops(isnotnull) "IS NOT NULL"
+  set unops(isnull)    "IS NULL"
+
+  set cols(0) a
+  set cols(1) b
+  set cols(2) c
+
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set cost 1000000.0
+      set ret [list]
+      set str [list]
+
+      set v 0
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable)} {
+          if {[info exists binops($C(op))]} {
+            lappend ret omit $i
+            lappend str "$cols($C(column)) $binops($C(op)) %$v%"
+            incr v
+            set cost [expr $cost / 2]
+          }
+          if {[info exists unops($C(op))]} {
+            lappend ret omit $i
+            lappend str "$cols($C(column)) $unops($C(op))"
+            incr v
+            set cost [expr $cost / 2]
+          }
+        }
+      }
+
+      lappend ret idxstr [join $str " AND "]
+      lappend ret cost $cost
+      return $ret
+    }
+
+    xFilter {
+      set q [lindex $args 1]
+      set a [lindex $args 2]
+      for {set v 0} {$v < [llength $a]} {incr v} {
+        set val [lindex $a $v]
+        set q [string map [list %$v% '$val'] $q]
+      }
+      if {$q==""} { set q 1 }
+      lappend ::xFilterQueries "WHERE $q"
+      return [list sql "SELECT rowid, * FROM t1x WHERE $q"]
+    }
+  }
+  return ""
+}
+
+proc vtab_simple {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t2(x)"
+    }
+    xBestIndex {
+      return [list cost 999999.0]
+    }
+    xFilter {
+      return [list sql "SELECT rowid, * FROM t2x"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+proc do_vtab_query_test {tn query result} {
+  set ::xFilterQueries [list]
+  uplevel [list
+    do_test $tn [string map [list %QUERY% $query] {
+      set r [execsql {%QUERY%}]
+      set r [concat $::xFilterQueries $r]
+      set r
+    }] [list {*}$result]
+  ]
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd');
+  CREATE TABLE t1x(a INTEGER, b TEXT, c REAL);
+  INSERT INTO t1x VALUES(1, 2, 3);
+  INSERT INTO t1x VALUES(4, 5, 6);
+  INSERT INTO t1x VALUES(7, 8, 9);
+
+  CREATE VIRTUAL TABLE t2 USING tcl('vtab_simple');
+  CREATE TABLE t2x(x INTEGER);
+  INSERT INTO t2x VALUES(1);
+}
+
+do_vtab_query_test 1.1 { SELECT * FROM t1 WHERE a!='hello'; } {
+  "WHERE a != 'hello'"
+  1 2 3.0 4 5 6.0 7 8 9.0
+}
+
+do_vtab_query_test 1.2.1 { SELECT * FROM t1 WHERE b!=8 } {
+  "WHERE b != '8'"
+  1 2 3.0 4 5 6.0
+}
+do_vtab_query_test 1.2.2 { SELECT * FROM t1 WHERE 8!=b } {
+  "WHERE b != '8'"
+  1 2 3.0 4 5 6.0
+}
+
+do_vtab_query_test 1.3 { SELECT * FROM t1 WHERE c IS NOT 3 } {
+  "WHERE c IS NOT '3'"
+  4 5 6.0 7 8 9.0
+}
+do_vtab_query_test 1.3.2 { SELECT * FROM t1 WHERE 3 IS NOT c } {
+  "WHERE c IS NOT '3'"
+  4 5 6.0 7 8 9.0
+}
+
+do_vtab_query_test 1.4.1 { SELECT * FROM t1, t2 WHERE x != a } {
+  "WHERE a != '1'"
+  4 5 6.0 1   7 8 9.0 1
+}
+do_vtab_query_test 1.4.2 { SELECT * FROM t1, t2 WHERE a != x } {
+  "WHERE a != '1'"
+  4 5 6.0 1   7 8 9.0 1
+}
+
+do_vtab_query_test 1.5.1 { SELECT * FROM t1 WHERE a IS NOT NULL } {
+  "WHERE a IS NOT NULL"
+  1 2 3.0 4 5 6.0 7 8 9.0
+}
+do_vtab_query_test 1.5.2 { SELECT * FROM t1 WHERE NULL IS NOT a } {
+  "WHERE a IS NOT ''"
+  1 2 3.0 4 5 6.0 7 8 9.0
+}
+
+do_vtab_query_test 1.6.1 { SELECT * FROM t1 WHERE a IS NULL } {
+  "WHERE a IS NULL"
+}
+
+do_vtab_query_test 1.6.2 { SELECT * FROM t1 WHERE NULL IS a } {
+  "WHERE a IS ''"
+}
+
+do_vtab_query_test 1.7.1 { SELECT * FROM t1 WHERE (a, b) IS (1, 2) } {
+  "WHERE a IS '1' AND b IS '2'"
+  1 2 3.0
+}
+do_vtab_query_test 1.7.2 { SELECT * FROM t1 WHERE (5, 4) IS (b, a) } {
+  {WHERE b IS '5' AND a IS '4'} 
+  4 5 6.0
+}
+
+finish_test
+

From 226cef4ef17f0a941d85426c5fa33fd86a82484f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 9 Sep 2017 20:38:49 +0000
Subject: [PATCH 073/270] Improved detection of malformed records by PRAGMA
 integrity_check.

FossilOrigin-Name: 8fa923caa1535fc9ebed0214d211fc3d09a015d78afd01f288c215a7980d25f2
---
 manifest      | 17 +++++----
 manifest.uuid |  2 +-
 src/pragma.c  | 98 +++++++++++++++++++++++++--------------------------
 3 files changed, 59 insertions(+), 58 deletions(-)

diff --git a/manifest b/manifest
index 4b71f03671..05498c7da4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC.
-D 2017-09-09T00:51:36.496
+C Improved\sdetection\sof\smalformed\srecords\sby\sPRAGMA\sintegrity_check.
+D 2017-09-09T20:38:49.243
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -450,7 +450,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
-F src/pragma.c faf5f9ec553b71035c63e08d31651bbd28bec826d546b32c8e9c295855ca3605
+F src/pragma.c 036a79afc71443ecfa15234e65dd7d519cc33cd2d1ec25f4e1d6657f19ac5d1a
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -1652,7 +1652,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e1e3ca7ea43a68b9b57dc38d8855f63b63a53feb8128b666a1becf87a2c70341
-R 14621bd82abfa0f650a4c6242c2108d7
-U mistachkin
-Z 11f83c8a0999c8fee516d7917728daf2
+P faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
+R c0b0efe9ffccc3828c7a4126e046d589
+T *branch * improved-integrity-check
+T *sym-improved-integrity-check *
+T -sym-trunk *
+U drh
+Z 18fb8b540ef1d907c45078ed2e61ad8c
diff --git a/manifest.uuid b/manifest.uuid
index ce2953ddb7..6c229b5b2c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
\ No newline at end of file
+8fa923caa1535fc9ebed0214d211fc3d09a015d78afd01f288c215a7980d25f2
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index fe1328c5ce..b3661b712c 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1549,12 +1549,6 @@ void sqlite3Pragma(
         int r1 = -1;
 
         if( pTab->tnum<1 ) continue;  /* Skip VIEWs or VIRTUAL TABLEs */
-        if( pTab->pCheck==0
-         && (pTab->tabFlags & TF_HasNotNull)==0
-         && (pTab->pIndex==0 || isQuick)
-        ){
-          continue;  /* No additional checks needed for this table */
-        }
         pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab);
         sqlite3ExprCacheClear(pParse);
         sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0,
@@ -1583,7 +1577,6 @@ void sqlite3Pragma(
           sqlite3VdbeJumpHere(v, jmp2);
         }
         /* Verify CHECK constraints */
-        sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3);
         if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){
           ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0);
           if( db->mallocFailed==0 ){
@@ -1609,51 +1602,56 @@ void sqlite3Pragma(
           }
           sqlite3ExprListDelete(db, pCheck);
         }
-        /* Validate index entries for the current row */
-        for(j=0, pIdx=pTab->pIndex; pIdx && !isQuick; pIdx=pIdx->pNext, j++){
-          int jmp2, jmp3, jmp4, jmp5;
-          int ckUniq = sqlite3VdbeMakeLabel(v);
-          if( pPk==pIdx ) continue;
-          r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3,
-                                       pPrior, r1);
-          pPrior = pIdx;
-          sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);  /* increment entry count */
-          /* Verify that an index entry exists for the current table row */
-          jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1,
-                                      pIdx->nColumn); VdbeCoverage(v);
-          sqlite3VdbeLoadString(v, 3, "row ");
-          sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3);
-          sqlite3VdbeLoadString(v, 4, " missing from index ");
-          sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
-          jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName);
-          sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
-          jmp4 = integrityCheckResultRow(v, 3);
-          sqlite3VdbeJumpHere(v, jmp2);
-          /* For UNIQUE indexes, verify that only one entry exists with the
-          ** current key.  The entry is unique if (1) any column is NULL
-          ** or (2) the next entry has a different key */
-          if( IsUniqueIndex(pIdx) ){
-            int uniqOk = sqlite3VdbeMakeLabel(v);
-            int jmp6;
-            int kk;
-            for(kk=0; kk<pIdx->nKeyCol; kk++){
-              int iCol = pIdx->aiColumn[kk];
-              assert( iCol!=XN_ROWID && iCol<pTab->nCol );
-              if( iCol>=0 && pTab->aCol[iCol].notNull ) continue;
-              sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk);
-              VdbeCoverage(v);
+        if( !isQuick ){ /* Omit the remaining tests for quick_check */
+          /* Sanity check on record header decoding */
+          sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3);
+          sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG);
+          /* Validate index entries for the current row */
+          for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
+            int jmp2, jmp3, jmp4, jmp5;
+            int ckUniq = sqlite3VdbeMakeLabel(v);
+            if( pPk==pIdx ) continue;
+            r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3,
+                                         pPrior, r1);
+            pPrior = pIdx;
+            sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */
+            /* Verify that an index entry exists for the current table row */
+            jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1,
+                                        pIdx->nColumn); VdbeCoverage(v);
+            sqlite3VdbeLoadString(v, 3, "row ");
+            sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3);
+            sqlite3VdbeLoadString(v, 4, " missing from index ");
+            sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
+            jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName);
+            sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
+            jmp4 = integrityCheckResultRow(v, 3);
+            sqlite3VdbeJumpHere(v, jmp2);
+            /* For UNIQUE indexes, verify that only one entry exists with the
+            ** current key.  The entry is unique if (1) any column is NULL
+            ** or (2) the next entry has a different key */
+            if( IsUniqueIndex(pIdx) ){
+              int uniqOk = sqlite3VdbeMakeLabel(v);
+              int jmp6;
+              int kk;
+              for(kk=0; kk<pIdx->nKeyCol; kk++){
+                int iCol = pIdx->aiColumn[kk];
+                assert( iCol!=XN_ROWID && iCol<pTab->nCol );
+                if( iCol>=0 && pTab->aCol[iCol].notNull ) continue;
+                sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk);
+                VdbeCoverage(v);
+              }
+              jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v);
+              sqlite3VdbeGoto(v, uniqOk);
+              sqlite3VdbeJumpHere(v, jmp6);
+              sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1,
+                                   pIdx->nKeyCol); VdbeCoverage(v);
+              sqlite3VdbeLoadString(v, 3, "non-unique entry in index ");
+              sqlite3VdbeGoto(v, jmp5);
+              sqlite3VdbeResolveLabel(v, uniqOk);
             }
-            jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v);
-            sqlite3VdbeGoto(v, uniqOk);
-            sqlite3VdbeJumpHere(v, jmp6);
-            sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1,
-                                 pIdx->nKeyCol); VdbeCoverage(v);
-            sqlite3VdbeLoadString(v, 3, "non-unique entry in index ");
-            sqlite3VdbeGoto(v, jmp5);
-            sqlite3VdbeResolveLabel(v, uniqOk);
+            sqlite3VdbeJumpHere(v, jmp4);
+            sqlite3ResolvePartIdxLabel(pParse, jmp3);
           }
-          sqlite3VdbeJumpHere(v, jmp4);
-          sqlite3ResolvePartIdxLabel(pParse, jmp3);
         }
         sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v);
         sqlite3VdbeJumpHere(v, loopTop-1);

From c2c6fd18ddce9929d870982d7f0b53a079316ed4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 9 Sep 2017 22:46:56 +0000
Subject: [PATCH 074/270] Simplification and performance improvement to
 sqlite3_reset().

FossilOrigin-Name: b6425d0170721d803a055a958f1823c9c4be925cd93ac47562ff723daf8ce2ed
---
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 src/vdbeaux.c | 40 +++++++++++++---------------------------
 3 files changed, 20 insertions(+), 37 deletions(-)

diff --git a/manifest b/manifest
index 05498c7da4..36d4ba2c2e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sdetection\sof\smalformed\srecords\sby\sPRAGMA\sintegrity_check.
-D 2017-09-09T20:38:49.243
+C Simplification\sand\sperformance\simprovement\sto\ssqlite3_reset().
+D 2017-09-09T22:46:56.684
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -529,7 +529,7 @@ F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 2dff8186d34e0e525838f77b2fac70bd480f0273a77015bf21b6a08f2792da82
+F src/vdbeaux.c 41d90002d774e234f95d1298fc70d25e3420e8d9ebd65aa270a8b771c5525790
 F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
 F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1652,10 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
-R c0b0efe9ffccc3828c7a4126e046d589
-T *branch * improved-integrity-check
-T *sym-improved-integrity-check *
-T -sym-trunk *
+P 8fa923caa1535fc9ebed0214d211fc3d09a015d78afd01f288c215a7980d25f2
+R 8957c035c2461edf5598d333c8c16670
 U drh
-Z 18fb8b540ef1d907c45078ed2e61ad8c
+Z 241aa0e3b67611bf0f414b4c1b2a7410
diff --git a/manifest.uuid b/manifest.uuid
index 6c229b5b2c..ac64dbf0f8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8fa923caa1535fc9ebed0214d211fc3d09a015d78afd01f288c215a7980d25f2
\ No newline at end of file
+b6425d0170721d803a055a958f1823c9c4be925cd93ac47562ff723daf8ce2ed
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 6de0efcb9e..e5465d16e7 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -2158,27 +2158,6 @@ static void closeAllCursors(Vdbe *p){
   assert( p->pAuxData==0 );
 }
 
-/*
-** Clean up the VM after a single run.
-*/
-static void Cleanup(Vdbe *p){
-  sqlite3 *db = p->db;
-
-#ifdef SQLITE_DEBUG
-  /* Execute assert() statements to ensure that the Vdbe.apCsr[] and 
-  ** Vdbe.aMem[] arrays have already been cleaned up.  */
-  int i;
-  if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 );
-  if( p->aMem ){
-    for(i=0; i<p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
-  }
-#endif
-
-  sqlite3DbFree(db, p->zErrMsg);
-  p->zErrMsg = 0;
-  p->pResultSet = 0;
-}
-
 /*
 ** Set the number of result columns that will be returned by this SQL
 ** statement. This is now set at compile time, rather than during
@@ -2904,8 +2883,6 @@ int sqlite3VdbeReset(Vdbe *p){
   if( p->pc>=0 ){
     vdbeInvokeSqllog(p);
     sqlite3VdbeTransferError(p);
-    sqlite3DbFree(db, p->zErrMsg);
-    p->zErrMsg = 0;
     if( p->runOnlyOnce ) p->expired = 1;
   }else if( p->rc && p->expired ){
     /* The expired flag was set on the VDBE before the first call
@@ -2913,13 +2890,22 @@ int sqlite3VdbeReset(Vdbe *p){
     ** called), set the database error in this case as well.
     */
     sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg);
-    sqlite3DbFree(db, p->zErrMsg);
-    p->zErrMsg = 0;
   }
 
-  /* Reclaim all memory used by the VDBE
+  /* Reset register contents and reclaim error message memory.
   */
-  Cleanup(p);
+#ifdef SQLITE_DEBUG
+  /* Execute assert() statements to ensure that the Vdbe.apCsr[] and 
+  ** Vdbe.aMem[] arrays have already been cleaned up.  */
+  int i;
+  if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 );
+  if( p->aMem ){
+    for(i=0; i<p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
+  }
+#endif
+  sqlite3DbFree(db, p->zErrMsg);
+  p->zErrMsg = 0;
+  p->pResultSet = 0;
 
   /* Save profiling information from this VDBE run.
   */

From 9ecd7086c1dfcf676063d4d31a621d0e18bcd037 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 10 Sep 2017 01:06:05 +0000
Subject: [PATCH 075/270] Simplifications to the PRAGMA integrity_check code
 generator.

FossilOrigin-Name: 99b9140c795c0c17c4e9d2547477c9dac056edfd443f2b2bd70edecd43c49ab7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pragma.c  | 48 +++++++++++++++++++++++++-----------------------
 3 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index 36d4ba2c2e..77107bb7d2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sand\sperformance\simprovement\sto\ssqlite3_reset().
-D 2017-09-09T22:46:56.684
+C Simplifications\sto\sthe\sPRAGMA\sintegrity_check\scode\sgenerator.
+D 2017-09-10T01:06:05.406
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -450,7 +450,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
-F src/pragma.c 036a79afc71443ecfa15234e65dd7d519cc33cd2d1ec25f4e1d6657f19ac5d1a
+F src/pragma.c 3799864604c6ac706b7423eab2b6e59ec404559f347af505cf8fe8f2e3a7b509
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8fa923caa1535fc9ebed0214d211fc3d09a015d78afd01f288c215a7980d25f2
-R 8957c035c2461edf5598d333c8c16670
+P b6425d0170721d803a055a958f1823c9c4be925cd93ac47562ff723daf8ce2ed
+R aac8ac34594edb6971d5273ddcaef89f
 U drh
-Z 241aa0e3b67611bf0f414b4c1b2a7410
+Z f7ba7422ef3914964199f82bd5c4f921
diff --git a/manifest.uuid b/manifest.uuid
index ac64dbf0f8..d243f46ffd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b6425d0170721d803a055a958f1823c9c4be925cd93ac47562ff723daf8ce2ed
\ No newline at end of file
+99b9140c795c0c17c4e9d2547477c9dac056edfd443f2b2bd70edecd43c49ab7
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index b3661b712c..9e7dc2be24 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -298,16 +298,16 @@ static const PragmaName *pragmaLocate(const char *zName){
 /*
 ** Helper subroutine for PRAGMA integrity_check:
 **
-** Generate code to output a single-column result row with the result
-** held in register regResult.  Decrement the result count and halt if
-** the maximum number of result rows have been issued.
+** Generate code to output a single-column result row with a value of the
+** string held in register 3.  Decrement the result count in register 1
+** and halt if the maximum number of result rows have been issued.
 */
-static int integrityCheckResultRow(Vdbe *v, int regResult){
+static int integrityCheckResultRow(Vdbe *v){
   int addr;
-  sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 1);
+  sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1);
   addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1);
   VdbeCoverage(v);
-  sqlite3VdbeAddOp2(v, OP_Halt, 0, 0);
+  sqlite3VdbeAddOp0(v, OP_Halt);
   return addr;
 }
 
@@ -1484,12 +1484,11 @@ void sqlite3Pragma(
 
     /* Do an integrity check on each database file */
     for(i=0; i<db->nDb; i++){
-      HashElem *x;
-      Hash *pTbls;
-      int *aRoot;
-      int cnt = 0;
-      int mxIdx = 0;
-      int nIdx;
+      HashElem *x;     /* For looping over tables in the schema */
+      Hash *pTbls;     /* Set of all tables in the schema */
+      int *aRoot;      /* Array of root page numbers of all btrees */
+      int cnt = 0;     /* Number of entries in aRoot[] */
+      int mxIdx = 0;   /* Maximum number of indexes for any table */
 
       if( OMIT_TEMPDB && i==1 ) continue;
       if( iDb>=0 && i!=iDb ) continue;
@@ -1504,8 +1503,9 @@ void sqlite3Pragma(
       assert( sqlite3SchemaMutexHeld(db, i, 0) );
       pTbls = &db->aDb[i].pSchema->tblHash;
       for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){
-        Table *pTab = sqliteHashData(x);
-        Index *pIdx;
+        Table *pTab = sqliteHashData(x);  /* Current table */
+        Index *pIdx;                      /* An index on pTab */
+        int nIdx;                         /* Number of indexes on pTab */
         if( HasRowid(pTab) ) cnt++;
         for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; }
         if( nIdx>mxIdx ) mxIdx = nIdx;
@@ -1533,9 +1533,8 @@ void sqlite3Pragma(
       sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0,
          sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName),
          P4_DYNAMIC);
-      sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1);
-      sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2);
-      integrityCheckResultRow(v, 2);
+      sqlite3VdbeAddOp3(v, OP_Concat, 2, 3, 3);
+      integrityCheckResultRow(v);
       sqlite3VdbeJumpHere(v, addr);
 
       /* Make sure all the indices are constructed correctly.
@@ -1553,6 +1552,9 @@ void sqlite3Pragma(
         sqlite3ExprCacheClear(pParse);
         sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0,
                                    1, 0, &iDataCur, &iIdxCur);
+        /* reg[7] counts the number of entries in the table.
+        ** reg[8+i] counts the number of entries in the i-th index 
+        */
         sqlite3VdbeAddOp2(v, OP_Integer, 0, 7);
         for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
           sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */
@@ -1573,7 +1575,7 @@ void sqlite3Pragma(
           zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName,
                               pTab->aCol[j].zName);
           sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
-          integrityCheckResultRow(v, 3);
+          integrityCheckResultRow(v);
           sqlite3VdbeJumpHere(v, jmp2);
         }
         /* Verify CHECK constraints */
@@ -1596,7 +1598,7 @@ void sqlite3Pragma(
             zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s",
                 pTab->zName);
             sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
-            integrityCheckResultRow(v, 3);
+            integrityCheckResultRow(v);
             sqlite3VdbeResolveLabel(v, addrCkOk);
             sqlite3ExprCachePop(pParse);
           }
@@ -1624,7 +1626,7 @@ void sqlite3Pragma(
             sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
             jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName);
             sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3);
-            jmp4 = integrityCheckResultRow(v, 3);
+            jmp4 = integrityCheckResultRow(v);
             sqlite3VdbeJumpHere(v, jmp2);
             /* For UNIQUE indexes, verify that only one entry exists with the
             ** current key.  The entry is unique if (1) any column is NULL
@@ -1663,9 +1665,9 @@ void sqlite3Pragma(
             sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3);
             addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v);
             sqlite3VdbeChangeP5(v, SQLITE_NOTNULL);
-            sqlite3VdbeLoadString(v, 3, pIdx->zName);
-            sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7);
-            integrityCheckResultRow(v, 7);
+            sqlite3VdbeLoadString(v, 4, pIdx->zName);
+            sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3);
+            integrityCheckResultRow(v);
             sqlite3VdbeJumpHere(v, addr);
           }
         }

From 09ccaf72d1782d1cb7b2d58bfba1ffd1f45a4841 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 11 Sep 2017 08:53:54 +0000
Subject: [PATCH 076/270] Add tests to check that affinities work with !=
 operators on virtual table column values. No changes to code.

FossilOrigin-Name: 8d24e0803d180448e637e33030a4ebf2c20eb1cab1074290c08f2eee0a23f66c
---
 manifest             | 15 +++++-------
 manifest.uuid        |  2 +-
 test/bestindex5.test | 58 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 3e1369face..a757538a5d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\svtab\sinterface\sto\shandle\sIS,\s!=,\sIS\sNOT,\sIS\sNULL\sand\sIS\sNOT\sNULL\nconstraints.
-D 2017-09-09T19:41:12.986
+C Add\stests\sto\scheck\sthat\saffinities\swork\swith\s!=\soperators\son\svirtual\stable\ncolumn\svalues.\sNo\schanges\sto\scode.
+D 2017-09-11T08:53:54.105
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -610,7 +610,7 @@ F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
 F test/bestindex3.test 578b6a52dab819e63f28e3640e04b32c85aed320
 F test/bestindex4.test 4cb5ff7dbaebadb87d366f51969271778423b455
-F test/bestindex5.test a7f1c32dc21d5c85afd4e9611e1160247107387b85a371fded36852c1c4959e0
+F test/bestindex5.test 412b42f8036b28d8b2f3534d89389ad946a4b1a65a12263f51936f7424296f1b
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
@@ -1653,10 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6
-R 71eba3dbcd321cff2772c9856db2cb4e
-T *branch * vtab-extra-ops
-T *sym-vtab-extra-ops *
-T -sym-trunk *
+P 34c8e952616013deb6fffec701ac5989afac9bef1bf92458a2e4ba92c7ee924f
+R 6bfa24cd8fec89ac45bb8a73348298d3
 U dan
-Z f5cb136cb2d9007b3611d202913746bf
+Z 91aa744c0aa50af3f54f9b98e2cb4c5c
diff --git a/manifest.uuid b/manifest.uuid
index e02149a44e..e6679f9a15 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-34c8e952616013deb6fffec701ac5989afac9bef1bf92458a2e4ba92c7ee924f
\ No newline at end of file
+8d24e0803d180448e637e33030a4ebf2c20eb1cab1074290c08f2eee0a23f66c
\ No newline at end of file
diff --git a/test/bestindex5.test b/test/bestindex5.test
index 237041f56f..ea93a47662 100644
--- a/test/bestindex5.test
+++ b/test/bestindex5.test
@@ -188,5 +188,63 @@ do_vtab_query_test 1.7.2 { SELECT * FROM t1 WHERE (5, 4) IS (b, a) } {
   4 5 6.0
 }
 
+#---------------------------------------------------------------------
+do_execsql_test 2.0.0 {
+  DELETE FROM t1x;
+  INSERT INTO t1x VALUES('a', 'b', 'c');
+}
+do_execsql_test 2.0.1 { SELECT * FROM t1 } {a b c}
+do_execsql_test 2.0.2 { SELECT * FROM t1 WHERE (a, b) != ('a', 'b'); } {}
+
+do_execsql_test 2.1.0 {
+  DELETE FROM t1x;
+  INSERT INTO t1x VALUES(7, 8, 9);
+}
+do_execsql_test 2.1.1 { SELECT * FROM t1 } {7 8 9.0}
+do_execsql_test 2.1.2 { SELECT * FROM t1 WHERE (a, b) != (7, '8') } {}
+do_execsql_test 2.1.3 { SELECT * FROM t1 WHERE a!=7 OR b!='8' }
+do_execsql_test 2.1.4 { SELECT * FROM t1 WHERE a!=7 OR b!='8' }
+
+
+do_execsql_test 2.2.1 {
+  CREATE TABLE t3(a INTEGER, b TEXT);
+  INSERT INTO t3 VALUES(45, 46);
+}
+do_execsql_test 2.2.2 { SELECT * FROM t3 WHERE (a, b) != (45, 46); }
+do_execsql_test 2.2.3 { SELECT * FROM t3 WHERE (a, b) != ('45', '46'); }
+do_execsql_test 2.2.4 { SELECT * FROM t3 WHERE (a, b) == (45, 46); } {45 46}
+do_execsql_test 2.2.5 { SELECT * FROM t3 WHERE (a, b) == ('45', '46'); } {45 46}
+
+#---------------------------------------------------------------------
+# Test the != operator on a virtual table with column affinities.
+#
+proc vtab_simple_integer {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t4(x INTEGER)"
+    }
+    xBestIndex {
+      return [list cost 999999.0]
+    }
+    xFilter {
+      return [list sql "SELECT rowid, * FROM t4x"]
+    }
+  }
+  return ""
+}
+
+do_execsql_test 3.0 {
+  CREATE TABLE t4x(a INTEGER);
+  INSERT INTO t4x VALUES(245);
+  CREATE VIRTUAL TABLE t4 USING tcl('vtab_simple_integer');
+}
+do_execsql_test 3.1 { SELECT rowid, * FROM t4 WHERE x=245; } {1 245}
+do_execsql_test 3.2 { SELECT rowid, * FROM t4 WHERE x='245'; } {1 245}
+do_execsql_test 3.3 { SELECT rowid, * FROM t4 WHERE x!=245; } {}
+do_execsql_test 3.4 { SELECT rowid, * FROM t4 WHERE x!='245'; } {}
+
+do_execsql_test 3.5 { SELECT rowid, * FROM t4 WHERE rowid!=1 OR x!='245'; } {}
+
+
 finish_test
 

From 33892c11a77cbe9688a0f7dba0624a83e10979dd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 11 Sep 2017 18:37:44 +0000
Subject: [PATCH 077/270] Minor adjustments to indentation and spacing for
 clarity.  No changes to code.

FossilOrigin-Name: d3153abda6fd48015b007081734738b9cc4622e3564dc78a7a470a72dc2ca236
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 18 +++++++++---------
 src/where.c     |  4 ++--
 4 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index a757538a5d..282bd770b7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sto\scheck\sthat\saffinities\swork\swith\s!=\soperators\son\svirtual\stable\ncolumn\svalues.\sNo\schanges\sto\scode.
-D 2017-09-11T08:53:54.105
+C Minor\sadjustments\sto\sindentation\sand\sspacing\sfor\sclarity.\s\sNo\schanges\sto\scode.
+D 2017-09-11T18:37:44.991
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -460,7 +460,7 @@ F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
-F src/sqlite.h.in d0ab3cae93cc9819f9e7ba5c8c8e3708e657c6cdbc61ecfa7dfadd19c0308ffa
+F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
 F src/sqliteInt.h f9ae3609a583aa51712083e1d5817f62c7d97c0a203036a9a62c439059e8258b
@@ -539,7 +539,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 7cc9692dc4f270f5a196d33d2ee1011ce6218a6061b73df592771a404ee3500c
+F src/where.c 3db249dfcef40357a295bc481af99cfcadc896d4eef259992681534db909e335
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c ffc3c90f68ad28c6eca1c8b05029f361bc151187be578985d992788d31f295ae
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 34c8e952616013deb6fffec701ac5989afac9bef1bf92458a2e4ba92c7ee924f
-R 6bfa24cd8fec89ac45bb8a73348298d3
-U dan
-Z 91aa744c0aa50af3f54f9b98e2cb4c5c
+P 8d24e0803d180448e637e33030a4ebf2c20eb1cab1074290c08f2eee0a23f66c
+R 0a1fb7fe2a3156d2459600f96873dd1d
+U drh
+Z 68e910dc75889ef64693d78040f26d24
diff --git a/manifest.uuid b/manifest.uuid
index e6679f9a15..239259e8a6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8d24e0803d180448e637e33030a4ebf2c20eb1cab1074290c08f2eee0a23f66c
\ No newline at end of file
+d3153abda6fd48015b007081734738b9cc4622e3564dc78a7a470a72dc2ca236
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index e2edfe4605..9ff366304f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6247,15 +6247,15 @@ struct sqlite3_index_info {
 ** an operator that is part of a constraint term in the wHERE clause of
 ** a query that uses a [virtual table].
 */
-#define SQLITE_INDEX_CONSTRAINT_EQ      2
-#define SQLITE_INDEX_CONSTRAINT_GT      4
-#define SQLITE_INDEX_CONSTRAINT_LE      8
-#define SQLITE_INDEX_CONSTRAINT_LT     16
-#define SQLITE_INDEX_CONSTRAINT_GE     32
-#define SQLITE_INDEX_CONSTRAINT_MATCH  64
-#define SQLITE_INDEX_CONSTRAINT_LIKE   65
-#define SQLITE_INDEX_CONSTRAINT_GLOB   66
-#define SQLITE_INDEX_CONSTRAINT_REGEXP 67
+#define SQLITE_INDEX_CONSTRAINT_EQ         2
+#define SQLITE_INDEX_CONSTRAINT_GT         4
+#define SQLITE_INDEX_CONSTRAINT_LE         8
+#define SQLITE_INDEX_CONSTRAINT_LT        16
+#define SQLITE_INDEX_CONSTRAINT_GE        32
+#define SQLITE_INDEX_CONSTRAINT_MATCH     64
+#define SQLITE_INDEX_CONSTRAINT_LIKE      65
+#define SQLITE_INDEX_CONSTRAINT_GLOB      66
+#define SQLITE_INDEX_CONSTRAINT_REGEXP    67
 #define SQLITE_INDEX_CONSTRAINT_NE        68
 #define SQLITE_INDEX_CONSTRAINT_ISNOT     69
 #define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70
diff --git a/src/where.c b/src/where.c
index 0e908e1516..a396b7db3b 100644
--- a/src/where.c
+++ b/src/where.c
@@ -953,8 +953,8 @@ static sqlite3_index_info *allocateIndexInfo(
       assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
 
       if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
-          && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
-        ){
+       && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
+      ){
         if( i<16 ) mNoOmit |= (1 << i);
         if( op==WO_LT ) pIdxCons[j].op = WO_LE;
         if( op==WO_GT ) pIdxCons[j].op = WO_GE;

From 303a69b5ad01c2a9d49b874ecba6069569ce1136 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 11 Sep 2017 19:47:37 +0000
Subject: [PATCH 078/270] Refactor names of constants and functions associated
 with the auxiliary operators for xBestIndex.

FossilOrigin-Name: 0fb992af669e765760a94d2b6e2330b21a98f39f18445a94b0a2290560368d58
---
 manifest        | 16 +++++++-------
 manifest.uuid   |  2 +-
 src/where.c     |  5 ++---
 src/whereInt.h  |  3 +--
 src/whereexpr.c | 56 ++++++++++++++++++++++++++-----------------------
 5 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/manifest b/manifest
index 282bd770b7..7b954551fb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\sadjustments\sto\sindentation\sand\sspacing\sfor\sclarity.\s\sNo\schanges\sto\scode.
-D 2017-09-11T18:37:44.991
+C Refactor\snames\sof\sconstants\sand\sfunctions\sassociated\swith\sthe\sauxiliary\noperators\sfor\sxBestIndex.
+D 2017-09-11T19:47:37.315
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -539,10 +539,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 3db249dfcef40357a295bc481af99cfcadc896d4eef259992681534db909e335
-F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
+F src/where.c d97f97ca6017747b7361af7f168a640d667c41cd1d4e91c2c5b2bd7538815d07
+F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
-F src/whereexpr.c ffc3c90f68ad28c6eca1c8b05029f361bc151187be578985d992788d31f295ae
+F src/whereexpr.c 4953ca4e769c047d0a00a1ba9085849626b1f3a6e89f6befcf5c38fa0722acdd
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8d24e0803d180448e637e33030a4ebf2c20eb1cab1074290c08f2eee0a23f66c
-R 0a1fb7fe2a3156d2459600f96873dd1d
+P d3153abda6fd48015b007081734738b9cc4622e3564dc78a7a470a72dc2ca236
+R 6bf300e1c6f9b6f8c5e667046a0485fa
 U drh
-Z 68e910dc75889ef64693d78040f26d24
+Z 6c35ef01413cc8966446242df55841d2
diff --git a/manifest.uuid b/manifest.uuid
index 239259e8a6..638546be3c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3153abda6fd48015b007081734738b9cc4622e3564dc78a7a470a72dc2ca236
\ No newline at end of file
+0fb992af669e765760a94d2b6e2330b21a98f39f18445a94b0a2290560368d58
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index a396b7db3b..7b806a7b6c 100644
--- a/src/where.c
+++ b/src/where.c
@@ -931,7 +931,7 @@ static sqlite3_index_info *allocateIndexInfo(
     pIdxCons[j].iTermOffset = i;
     op = pTerm->eOperator & WO_ALL;
     if( op==WO_IN ) op = WO_EQ;
-    if( op==WO_MATCH ){
+    if( op==WO_AUX ){
       pIdxCons[j].op = pTerm->eMatchOp;
     }else if( op & (WO_ISNULL|WO_IS) ){
       if( op==WO_ISNULL ){
@@ -949,8 +949,7 @@ static sqlite3_index_info *allocateIndexInfo(
       assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
       assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
       assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
-      assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
-      assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
+      assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
 
       if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
        && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
diff --git a/src/whereInt.h b/src/whereInt.h
index bac81da721..0aa7bc1da0 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -515,7 +515,6 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 **     WO_LE    == SQLITE_INDEX_CONSTRAINT_LE
 **     WO_GT    == SQLITE_INDEX_CONSTRAINT_GT
 **     WO_GE    == SQLITE_INDEX_CONSTRAINT_GE
-**     WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH
 */
 #define WO_IN     0x0001
 #define WO_EQ     0x0002
@@ -523,7 +522,7 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 #define WO_LE     (WO_EQ<<(TK_LE-TK_EQ))
 #define WO_GT     (WO_EQ<<(TK_GT-TK_EQ))
 #define WO_GE     (WO_EQ<<(TK_GE-TK_EQ))
-#define WO_MATCH  0x0040
+#define WO_AUX    0x0040       /* Op useful to virtual tables only */
 #define WO_IS     0x0080
 #define WO_ISNULL 0x0100
 #define WO_OR     0x0200       /* Two or more OR-connected terms */
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 45d3a7e33f..da07f56ce8 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -312,27 +312,32 @@ static int isLikeOrGlob(
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
-** Check to see if the given expression is of the form
+** Check to see if the pExpr expression is a form that needs to be passed
+** to the xBestIndex method of virtual tables.  Forms of interest include:
 **
-**         column OP expr
+**          Expression                   Virtual Table Operator
+**          -----------------------      ---------------------------------
+**      1.  column MATCH expr            SQLITE_INDEX_CONSTRAINT_MATCH
+**      2.  column GLOB expr             SQLITE_INDEX_CONSTRAINT_GLOB
+**      3.  column LIKE expr             SQLITE_INDEX_CONSTRAINT_LIKE
+**      4.  column REGEXP expr           SQLITE_INDEX_CONSTRAINT_REGEXP
+**      5.  column != expr               SQLITE_INDEX_CONSTRAINT_NE
+**      6.  expr != column               SQLITE_INDEX_CONSTRAINT_NE
+**      7.  column IS NOT expr           SQLITE_INDEX_CONSTRAINT_ISNOT
+**      8.  expr IS NOT column           SQLITE_INDEX_CONSTRAINT_ISNOT
+**      9.  column IS NOT NULL           SQLITE_INDEX_CONSTRAINT_ISNOTNULL
 **
-** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a 
-** column of a virtual table. If so, set *ppLeft to point to the
-** expression for "column", *ppRight to "expr" and return 1.
-**
-** Also check if the expression is one of:
-**
-**         column != expr
-**         column IS NOT expr
-**         column IS NOT NULL
-**
-** where "column" is a column of a virtual table. If so, set *ppLeft
-** to point to "column", *ppRight to "expr" and return 1. Or, if "expr"
-** is also a column of a virtual table, return 2.
+** In every case, "column" must be a column of a virtual table.  If there
+** is a match, set *ppLeft to the "column" expression, set *ppRight to the 
+** "expr" expression (even though in forms (6) and (8) the column is on the
+** right and the expression is on the left).  Also set *peOp2 to the
+** appropriate virtual table operator.  The return value is 1 or 2 if there
+** is a match.  The usual return is 1, but if the RHS is also a column
+** of virtual table in forms (5) or (7) then return 2.
 **
 ** If the expression matches none of the patterns above, return 0.
 */
-static int isMatchOfColumn(
+static int isAuxiliaryVtabOperator(
   Expr *pExpr,                    /* Test this expression */
   unsigned char *peOp2,           /* OUT: 0 for MATCH, or else an op2 value */
   Expr **ppLeft,                  /* Column expression to left of MATCH/op2 */
@@ -352,9 +357,6 @@ static int isMatchOfColumn(
     Expr *pCol;                     /* Column reference */
     int i;
 
-    if( pExpr->op!=TK_FUNCTION ){
-      return 0;
-    }
     pList = pExpr->x.pList;
     if( pList==0 || pList->nExpr!=2 ){
       return 0;
@@ -638,7 +640,7 @@ static void exprAnalyzeOrTerm(
           for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
             if( allowedOp(pAndTerm->pExpr->op) 
-             || pAndTerm->eOperator==WO_MATCH 
+             || pAndTerm->eOperator==WO_AUX
             ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
             }
@@ -1220,17 +1222,19 @@ static void exprAnalyze(
 #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  /* Add a WO_MATCH auxiliary term to the constraint set if the
-  ** current expression is of the form:  column MATCH expr.
+  /* Add a WO_AUX auxiliary term to the constraint set if the
+  ** current expression is of the form "column OP expr" where OP
+  ** is an operator that gets passed into virtual tables but which is
+  ** not normally optimized for ordinary tables.  In other words, OP
+  ** is one of MATCH, LIKE, GLOB, REGEXP, !=, IS, IS NOT, or NOT NULL.
   ** This information is used by the xBestIndex methods of
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
   if( pWC->op==TK_AND ){
     Expr *pRight, *pLeft;
-    int i;
-    int res = isMatchOfColumn(pExpr, &eOp2, &pLeft, &pRight);
-    for(i=0; i<res; i++){
+    int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight);
+    while( res-- > 0 ){
       int idxNew;
       WhereTerm *pNewTerm;
       Bitmask prereqColumn, prereqExpr;
@@ -1250,7 +1254,7 @@ static void exprAnalyze(
         pNewTerm->prereqRight = prereqExpr;
         pNewTerm->leftCursor = pLeft->iTable;
         pNewTerm->u.leftColumn = pLeft->iColumn;
-        pNewTerm->eOperator = WO_MATCH;
+        pNewTerm->eOperator = WO_AUX;
         pNewTerm->eMatchOp = eOp2;
         markTermAsChild(pWC, idxNew, idxTerm);
         pTerm = &pWC->a[idxTerm];

From 68262d85ec2b40fc6d976fa696071a64e8054906 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 11 Sep 2017 23:46:59 +0000
Subject: [PATCH 079/270] New test case for ticket [b899b6042f97f52d].

FossilOrigin-Name: 9d91ee5ee3f316a8804b02eb3c34b175b8f87b28c22df56f481af708c54e297c
---
 manifest         | 13 ++++++-------
 manifest.uuid    |  2 +-
 test/whereF.test | 12 ++++++++++++
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index c64290312a..681d32ff4e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\snew\soperators\sin\svirtual\stables:\s!=,\sIS,\sIS\sNOT,\sIS\sNULL,\sand\sNOTNULL.
-D 2017-09-11T20:54:54.464
+C New\stest\scase\sfor\sticket\s[b899b6042f97f52d].
+D 2017-09-11T23:46:59.120
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -1532,7 +1532,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test ec178da9a65f50d1fefeb0fd1303faa01fe74d1eec5b24db89b040b7c4faa9c7
+F test/whereF.test 7c426e0bf303407910c21f79d3a6742f7b33c9b27b2dddd230dfc8c2508981a6
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1653,8 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P faa22e29a5a05a16d46a428d838acedfa7d6fad6239495d59a6a1f4764e1b1b6 0fb992af669e765760a94d2b6e2330b21a98f39f18445a94b0a2290560368d58
-R 6bf300e1c6f9b6f8c5e667046a0485fa
-T +closed 0fb992af669e765760a94d2b6e2330b21a98f39f18445a94b0a2290560368d58
+P 3b7dbe9da90b7db336eed597a73a5364d8a691e57c1febd60908349f57a539ad
+R d1f8e4be016c31ab042e5cda9ad31b92
 U drh
-Z 831d5d4673d5f6c544cd75e0243e7e97
+Z d58faf0ad143f7a7b57c47c15bbc02fa
diff --git a/manifest.uuid b/manifest.uuid
index a1f6d564ce..f3ce3d7f92 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3b7dbe9da90b7db336eed597a73a5364d8a691e57c1febd60908349f57a539ad
\ No newline at end of file
+9d91ee5ee3f316a8804b02eb3c34b175b8f87b28c22df56f481af708c54e297c
\ No newline at end of file
diff --git a/test/whereF.test b/test/whereF.test
index 7b42e0a19f..a2361cfd16 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -184,6 +184,18 @@ ifcapable json1 {
     CREATE TABLE t6(x);
     SELECT * FROM t6 WHERE 1 IN (SELECT value FROM json_each(x));
   } {}
+
+  do_execsql_test 6.2 {
+    DROP TABLE t6;
+    CREATE TABLE t6(a,b,c);
+    INSERT INTO t6 VALUES
+     (0,null,'{"a":0,"b":[3,4,5],"c":{"x":4.5,"y":7.8}}'),
+     (1,null,'{"a":1,"b":[3,4,5],"c":{"x":4.5,"y":7.8}}'),
+     (2,null,'{"a":9,"b":[3,4,5],"c":{"x":4.5,"y":7.8}}');
+    SELECT * FROM t6
+     WHERE (EXISTS (SELECT 1 FROM json_each(t6.c) AS x WHERE x.value=1));
+  } {1 {} {{"a":1,"b":[3,4,5],"c":{"x":4.5,"y":7.8}}}}
+    
 }
 
 finish_test

From 3e1e14d818c335c4d3dbe6f80559c54e929c5399 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 12 Sep 2017 00:24:45 +0000
Subject: [PATCH 080/270] Make sure all connections are closed in the
 pragma.test script. Test change only - no changes to code.

FossilOrigin-Name: d7b7f038a2c5e81a58c1dd4b556096b58ce10c30eb74fd4b770445927fdb62f3
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/pragma.test |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 681d32ff4e..106c3708ae 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\stest\scase\sfor\sticket\s[b899b6042f97f52d].
-D 2017-09-11T23:46:59.120
+C Make\ssure\sall\sconnections\sare\sclosed\sin\sthe\spragma.test\sscript.\nTest\schange\sonly\s-\sno\schanges\sto\scode.
+D 2017-09-12T00:24:45.988
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -1096,7 +1096,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
 F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531
-F test/pragma.test faa8b64c4be28b67b3a0d3e9c977e2feff7dd2a4b08f2fd062a95d30f8c8fd1c
+F test/pragma.test 277488e8098e55eef32cf3188513269035cccf5a1804d505db7ac4c725fa52a5
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3b7dbe9da90b7db336eed597a73a5364d8a691e57c1febd60908349f57a539ad
-R d1f8e4be016c31ab042e5cda9ad31b92
+P 9d91ee5ee3f316a8804b02eb3c34b175b8f87b28c22df56f481af708c54e297c
+R 569dfff7f83956e010969f23270d7c7f
 U drh
-Z d58faf0ad143f7a7b57c47c15bbc02fa
+Z dc7e3abd878bac78055247ca067fef2c
diff --git a/manifest.uuid b/manifest.uuid
index f3ce3d7f92..2ece8022fb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9d91ee5ee3f316a8804b02eb3c34b175b8f87b28c22df56f481af708c54e297c
\ No newline at end of file
+d7b7f038a2c5e81a58c1dd4b556096b58ce10c30eb74fd4b770445927fdb62f3
\ No newline at end of file
diff --git a/test/pragma.test b/test/pragma.test
index cebbf9ddc0..1445410696 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -1932,6 +1932,7 @@ do_test 23.5 {
     PRAGMA foreign_key_list(t2);
   }
 } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE}
+db2 close
 
 reset_db
 do_execsql_test 24.0 {

From b9772e7fe8eed080490e22c838c2ddf07d0e7c6b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 12 Sep 2017 13:27:43 +0000
Subject: [PATCH 081/270] Changes to (hopefully) get the build working with
 recent Intel compilers.

FossilOrigin-Name: b22cdd6734ecda2b2b9749668f353abc2660f192d6a40c5d266309d30e25695e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/util.c    | 10 +++++++---
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 106c3708ae..02e9cbfbb7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sall\sconnections\sare\sclosed\sin\sthe\spragma.test\sscript.\nTest\schange\sonly\s-\sno\schanges\sto\scode.
-D 2017-09-12T00:24:45.988
+C Changes\sto\s(hopefully)\sget\sthe\sbuild\sworking\swith\srecent\sIntel\scompilers.
+D 2017-09-12T13:27:43.222
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -523,7 +523,7 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
 F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c 773843506ce694714bc96fe67c30c37015f90ef515d0e70f1f8d5c9c24088152
+F src/util.c c2e1f79f6aa027d67eeae82390c633597fa936679a0e63d03eaf8a6789f65783
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9d91ee5ee3f316a8804b02eb3c34b175b8f87b28c22df56f481af708c54e297c
-R 569dfff7f83956e010969f23270d7c7f
+P d7b7f038a2c5e81a58c1dd4b556096b58ce10c30eb74fd4b770445927fdb62f3
+R faaafb9d700186855b0c5f4648cf9c3d
 U drh
-Z dc7e3abd878bac78055247ca067fef2c
+Z b126c5cc9d8f4cb4e874c4cda7b85db8
diff --git a/manifest.uuid b/manifest.uuid
index 2ece8022fb..d853015a65 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d7b7f038a2c5e81a58c1dd4b556096b58ce10c30eb74fd4b770445927fdb62f3
\ No newline at end of file
+b22cdd6734ecda2b2b9749668f353abc2660f192d6a40c5d266309d30e25695e
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index a6d349296f..09f7e051a9 100644
--- a/src/util.c
+++ b/src/util.c
@@ -491,7 +491,11 @@ do_atof_calc:
           if( esign<0 ){
             result = 0.0*s;
           }else{
+#ifdef INFINITY
+            result = INFINITY;
+#else
             result = 1e308*1e308*s;  /* Infinity */
+#endif
           }
         }
       }else{
@@ -1276,7 +1280,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){
 ** overflow, leave *pA unchanged and return 1.
 */
 int sqlite3AddInt64(i64 *pA, i64 iB){
-#if GCC_VERSION>=5004000
+#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER)
   return __builtin_add_overflow(*pA, iB, pA);
 #else
   i64 iA = *pA;
@@ -1296,7 +1300,7 @@ int sqlite3AddInt64(i64 *pA, i64 iB){
 #endif
 }
 int sqlite3SubInt64(i64 *pA, i64 iB){
-#if GCC_VERSION>=5004000
+#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER)
   return __builtin_sub_overflow(*pA, iB, pA);
 #else
   testcase( iB==SMALLEST_INT64+1 );
@@ -1311,7 +1315,7 @@ int sqlite3SubInt64(i64 *pA, i64 iB){
 #endif
 }
 int sqlite3MulInt64(i64 *pA, i64 iB){
-#if GCC_VERSION>=5004000
+#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER)
   return __builtin_mul_overflow(*pA, iB, pA);
 #else
   i64 iA = *pA;

From 3ba18adde5d338f68997fcac611a877662fc7518 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 12 Sep 2017 15:05:34 +0000
Subject: [PATCH 082/270] Fix an error in [b22cdd67] that can cause a negative
 infinity to be (rarely) reported as a positive infinity.

FossilOrigin-Name: 9780b23ca375de6a542516fbc03eb39d5a393ca577718fda231d0d0ccf3b1c7e
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/util.c    |  2 +-
 test/nan.test |  8 +++++---
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 02e9cbfbb7..6c4f404802 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\s(hopefully)\sget\sthe\sbuild\sworking\swith\srecent\sIntel\scompilers.
-D 2017-09-12T13:27:43.222
+C Fix\san\serror\sin\s[b22cdd67]\sthat\scan\scause\sa\snegative\sinfinity\sto\sbe\s(rarely)\nreported\sas\sa\spositive\sinfinity.
+D 2017-09-12T15:05:34.004
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -523,7 +523,7 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
 F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c c2e1f79f6aa027d67eeae82390c633597fa936679a0e63d03eaf8a6789f65783
+F src/util.c ece10cb34b4a62cbb3aab8d711b67e93a2765e0b81d0f72458cb801a3ac60703
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
@@ -1057,7 +1057,7 @@ F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101
 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4
 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
-F test/nan.test dacc57f80859c06a433d30839336fe227d2038b3
+F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
 F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d7b7f038a2c5e81a58c1dd4b556096b58ce10c30eb74fd4b770445927fdb62f3
-R faaafb9d700186855b0c5f4648cf9c3d
+P b22cdd6734ecda2b2b9749668f353abc2660f192d6a40c5d266309d30e25695e
+R 39b328d722a14e00d87cd1b0274ba2b6
 U drh
-Z b126c5cc9d8f4cb4e874c4cda7b85db8
+Z 7a920600be1c7fd6aaa5dec418f9fad6
diff --git a/manifest.uuid b/manifest.uuid
index d853015a65..9386e9ab66 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b22cdd6734ecda2b2b9749668f353abc2660f192d6a40c5d266309d30e25695e
\ No newline at end of file
+9780b23ca375de6a542516fbc03eb39d5a393ca577718fda231d0d0ccf3b1c7e
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index 09f7e051a9..ebcc5bec6e 100644
--- a/src/util.c
+++ b/src/util.c
@@ -492,7 +492,7 @@ do_atof_calc:
             result = 0.0*s;
           }else{
 #ifdef INFINITY
-            result = INFINITY;
+            result = INFINITY*s;
 #else
             result = 1e308*1e308*s;  /* Infinity */
 #endif
diff --git a/test/nan.test b/test/nan.test
index 27fa04a351..615a4ad227 100644
--- a/test/nan.test
+++ b/test/nan.test
@@ -366,8 +366,10 @@ do_realnum_test nan-4.35 {
   }
 } {0.0 real}
 
-  
-
-
+do_realnum_test nan-4.40 {
+  db eval {
+    SELECT cast('-1e999' AS real);
+  }
+} {-inf}
 
 finish_test

From bf60baa9856b292a09a6de832a9f76640ec1891f Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 12 Sep 2017 23:58:47 +0000
Subject: [PATCH 083/270] Remove use of the rand_s() function (added by
 [139081bef9f63c3e]) as it appears to cause issues with some third-party DLLs.

FossilOrigin-Name: 3a2793aa65727cbbf8c4678d478cf8fb02350f19ff98201934a984727368d64d
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/os_win.c    |  3 ---
 src/sqliteInt.h |  8 --------
 4 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 6c4f404802..49d41c7878 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\serror\sin\s[b22cdd67]\sthat\scan\scause\sa\snegative\sinfinity\sto\sbe\s(rarely)\nreported\sas\sa\spositive\sinfinity.
-D 2017-09-12T15:05:34.004
+C Remove\suse\sof\sthe\srand_s()\sfunction\s(added\sby\s[139081bef9f63c3e])\sas\sit\sappears\sto\scause\sissues\swith\ssome\sthird-party\sDLLs.
+D 2017-09-12T23:58:47.871
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -442,7 +442,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
-F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
+F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 967168bba88d2dc790ed9618bd4ba7bfe475b67b521ef6da305a6425c592928f
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -463,7 +463,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h f9ae3609a583aa51712083e1d5817f62c7d97c0a203036a9a62c439059e8258b
+F src/sqliteInt.h 2272cd09b7f137bb2423af589d636b5b741037db9b329d359c1ef6556fac5a4c
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b22cdd6734ecda2b2b9749668f353abc2660f192d6a40c5d266309d30e25695e
-R 39b328d722a14e00d87cd1b0274ba2b6
-U drh
-Z 7a920600be1c7fd6aaa5dec418f9fad6
+P 9780b23ca375de6a542516fbc03eb39d5a393ca577718fda231d0d0ccf3b1c7e
+R 342e5faaf3ce5ac225c0025f84f1d0f8
+U mistachkin
+Z ee0e295b03f3976f16bf7211fd8df318
diff --git a/manifest.uuid b/manifest.uuid
index 9386e9ab66..ed48465def 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9780b23ca375de6a542516fbc03eb39d5a393ca577718fda231d0d0ccf3b1c7e
\ No newline at end of file
+3a2793aa65727cbbf8c4678d478cf8fb02350f19ff98201934a984727368d64d
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 7045448fe1..f7c835a522 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -5696,9 +5696,6 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){
   EntropyGatherer e;
   UNUSED_PARAMETER(pVfs);
   memset(zBuf, 0, nBuf);
-#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE
-  rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */
-#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */
   e.a = (unsigned char*)zBuf;
   e.na = nBuf;
   e.nXor = 0;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 4ce54cf73d..58d64ba2f2 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -50,14 +50,6 @@
 #  define SQLITE_TCLAPI
 #endif
 
-/*
-** Make sure that rand_s() is available on Windows systems with MSVC 2005
-** or higher.
-*/
-#if defined(_MSC_VER) && _MSC_VER>=1400
-#  define _CRT_RAND_S
-#endif
-
 /*
 ** Include the header file used to customize the compiler options for MSVC.
 ** This should be done first so that it can successfully prevent spurious

From 77187faf5aa912981671a9d9b73aa9cc3e28fce6 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 13 Sep 2017 00:01:26 +0000
Subject: [PATCH 084/270] Fix test script comment.  No changes to code.

FossilOrigin-Name: a5a1949259a26c3330d7b1dcf38a8dd2e9b2c3af8d52f937e8537df138c9a55a
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 test/win32heap.test |  3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 49d41c7878..1e54213d5a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\suse\sof\sthe\srand_s()\sfunction\s(added\sby\s[139081bef9f63c3e])\sas\sit\sappears\sto\scause\sissues\swith\ssome\sthird-party\sDLLs.
-D 2017-09-12T23:58:47.871
+C Fix\stest\sscript\scomment.\s\sNo\schanges\sto\scode.
+D 2017-09-13T00:01:26.395
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -1541,7 +1541,7 @@ F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
-F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
+F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9780b23ca375de6a542516fbc03eb39d5a393ca577718fda231d0d0ccf3b1c7e
-R 342e5faaf3ce5ac225c0025f84f1d0f8
+P 3a2793aa65727cbbf8c4678d478cf8fb02350f19ff98201934a984727368d64d
+R 1b5d97330475c133bb2adcba21536957
 U mistachkin
-Z ee0e295b03f3976f16bf7211fd8df318
+Z e62aad7ac8dd39a60ddf24398d01019c
diff --git a/manifest.uuid b/manifest.uuid
index ed48465def..8af7c330d1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3a2793aa65727cbbf8c4678d478cf8fb02350f19ff98201934a984727368d64d
\ No newline at end of file
+a5a1949259a26c3330d7b1dcf38a8dd2e9b2c3af8d52f937e8537df138c9a55a
\ No newline at end of file
diff --git a/test/win32heap.test b/test/win32heap.test
index b92f8040e6..82a3f6b663 100644
--- a/test/win32heap.test
+++ b/test/win32heap.test
@@ -9,8 +9,7 @@
 #
 #***********************************************************************
 # This file implements regression tests for SQLite library.  The
-# focus of this script is recovery from transient manditory locks
-# that sometimes appear on database files due to anti-virus software.
+# focus of this script is the Win32 heap implementation.
 #
 
 if {$tcl_platform(platform)!="windows"} return

From 74588ceb317ad9d3b97aee0c1dfcca9f957dc01f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Sep 2017 00:13:05 +0000
Subject: [PATCH 085/270] PRAGMA integrity_check returns SQLITE_OK even if it
 encounters corruption while scanning indexes.

FossilOrigin-Name: 81f62e99f27dedf3dc10fa1593c4cd9fc158680867206aee853e503a0cc8df71
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/pragma.c     |  6 ++++++
 src/vdbe.c       | 20 +++++++++++++++-----
 test/pragma.test |  2 +-
 5 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 77107bb7d2..2141adaa03 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplifications\sto\sthe\sPRAGMA\sintegrity_check\scode\sgenerator.
-D 2017-09-10T01:06:05.406
+C PRAGMA\sintegrity_check\sreturns\sSQLITE_OK\seven\sif\sit\sencounters\scorruption\nwhile\sscanning\sindexes.
+D 2017-09-13T00:13:05.025
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -450,7 +450,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
-F src/pragma.c 3799864604c6ac706b7423eab2b6e59ec404559f347af505cf8fe8f2e3a7b509
+F src/pragma.c 88d99aa4752894aaf4102eefd09b5e497f4277661aff69f8fff61f8611f19d14
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -525,7 +525,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 773843506ce694714bc96fe67c30c37015f90ef515d0e70f1f8d5c9c24088152
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 82fc4553a0986a06bdd0d2b03a424e159bba5c74802fabb2841aa6cd27ccd962
+F src/vdbe.c 0a0ef7f0759ee4c3ed5ff0c6e9641216d4b31ebb0fea9a7b5a9c4a86ce875ff3
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -1095,7 +1095,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
 F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531
-F test/pragma.test faa8b64c4be28b67b3a0d3e9c977e2feff7dd2a4b08f2fd062a95d30f8c8fd1c
+F test/pragma.test 21b5a5c2e44121bb88a009b784b52c4e45b9d726c13963b1f6c38869970e0c95
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264
@@ -1652,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b6425d0170721d803a055a958f1823c9c4be925cd93ac47562ff723daf8ce2ed
-R aac8ac34594edb6971d5273ddcaef89f
+P 99b9140c795c0c17c4e9d2547477c9dac056edfd443f2b2bd70edecd43c49ab7
+R a246d27aca858c3472ea60336e5e6182
 U drh
-Z f7ba7422ef3914964199f82bd5c4f921
+Z 6a3c45b7e13d2d307b1bb08e00187e8b
diff --git a/manifest.uuid b/manifest.uuid
index d243f46ffd..4b4a5e69ad 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-99b9140c795c0c17c4e9d2547477c9dac056edfd443f2b2bd70edecd43c49ab7
\ No newline at end of file
+81f62e99f27dedf3dc10fa1593c4cd9fc158680867206aee853e503a0cc8df71
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index 9e7dc2be24..29c83480fc 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1681,6 +1681,9 @@ void sqlite3Pragma(
         { OP_IfNotZero,   1, 4,        0},    /* 1 */
         { OP_String8,     0, 3,        0},    /* 2 */
         { OP_ResultRow,   3, 1,        0},    /* 3 */
+        { OP_Halt,        0, 0,        0},    /* 4 */
+        { OP_String8,     0, 3,        0},    /* 5 */
+        { OP_Goto,        0, 3,        0},    /* 6 */
       };
       VdbeOp *aOp;
 
@@ -1689,7 +1692,10 @@ void sqlite3Pragma(
         aOp[0].p2 = 1-mxErr;
         aOp[2].p4type = P4_STATIC;
         aOp[2].p4.z = "ok";
+        aOp[5].p4type = P4_STATIC;
+        aOp[5].p4.z = (char*)sqlite3ErrStr(SQLITE_CORRUPT);
       }
+      sqlite3VdbeChangeP3(v, 0, sqlite3VdbeCurrentAddr(v)-2);
     }
   }
   break;
diff --git a/src/vdbe.c b/src/vdbe.c
index 70e745afe0..37699bbba0 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2461,8 +2461,7 @@ case OP_Column: {
       ** extra bytes for the header length itself.  32768*3 + 3 = 98307.
       */
       if( aOffset[0] > 98307 || aOffset[0] > pC->payloadSize ){
-        rc = SQLITE_CORRUPT_BKPT;
-        goto abort_due_to_error;
+        goto op_column_corrupt;
       }
     }else{
       /* This is an optimization.  By skipping over the first few tests
@@ -2535,8 +2534,7 @@ case OP_Column: {
           zHdr = zEndHdr;
         }else{
           if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
-          rc = SQLITE_CORRUPT_BKPT;
-          goto abort_due_to_error;
+          goto op_column_corrupt;
         }
       }
 
@@ -2631,6 +2629,15 @@ op_column_out:
   UPDATE_MAX_BLOBSIZE(pDest);
   REGISTER_TRACE(pOp->p3, pDest);
   break;
+
+op_column_corrupt:
+  if( aOp[0].p3>0 ){
+    pOp = &aOp[aOp[0].p3-1];
+    break;
+  }else{
+    rc = SQLITE_CORRUPT_BKPT;
+    goto abort_due_to_error;
+  }
 }
 
 /* Opcode: Affinity P1 P2 * P4 *
@@ -7015,7 +7022,7 @@ case OP_Function: {
 }
 
 
-/* Opcode: Init P1 P2 * P4 *
+/* Opcode: Init P1 P2 P3 P4 *
 ** Synopsis: Start at P2
 **
 ** Programs contain a single instance of this opcode as the very first
@@ -7029,6 +7036,9 @@ case OP_Function: {
 **
 ** Increment the value of P1 so that OP_Once opcodes will jump the
 ** first time they are evaluated for this run.
+**
+** If P3 is not zero, then it is an address to jump to if an SQLITE_CORRUPT
+** error is encountered.
 */
 case OP_Init: {          /* jump */
   char *zTrace;
diff --git a/test/pragma.test b/test/pragma.test
index cebbf9ddc0..c0be5f4b3d 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -1952,7 +1952,7 @@ do_catchsql_test 24.1 {
 } {1 {database disk image is malformed}}
 do_catchsql_test 24.2 {
   PRAGMA integrity_check;
-} {1 {database disk image is malformed}}
+} {0 {{database disk image is malformed}}}
 
 database_never_corrupt
 finish_test

From 4c0873573212609765c062775748b9d2f453040a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Sep 2017 12:55:59 +0000
Subject: [PATCH 086/270] Disable FTS3 matchinfo() tests that assume a
 littleEndian platform when running on bigEndian.

FossilOrigin-Name: 87ccdf9cbb9284553330683d4971be4f523ce922089aee6dffccfc18b3004263
---
 manifest           | 13 ++++---
 manifest.uuid      |  2 +-
 test/fts3conf.test | 84 ++++++++++++++++++++++++----------------------
 3 files changed, 50 insertions(+), 49 deletions(-)

diff --git a/manifest b/manifest
index 11fb3c0076..42cba1232c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\sto\sPRAGMA\sintegrity_check\sfor\sbetter\sdetection\sof\smalformed\nrecords.\s\sIntegrity_check\snow\savoids\sreturning\sSQLITE_CORRUPT\son\sa\scorrupt\nrecord.\s\sAlso\sincludes\smicrooptimizations\sthat\sI\sstumbled\sover\swhile\sworking\non\sintegrity_check.
-D 2017-09-13T00:33:36.268
+C Disable\sFTS3\smatchinfo()\stests\sthat\sassume\sa\slittleEndian\splatform\swhen\nrunning\son\sbigEndian.
+D 2017-09-13T12:55:59.555
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -835,7 +835,7 @@ F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba
 F test/fts3b.test c15c4a9d04e210d0be67e54ce6a87b927168fbf9c1e3faec8c1a732c366fd491
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c
-F test/fts3conf.test 60317efd562080e198b5bdc9fcd222ce32cf01d7
+F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f5a76b
 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3corrupt3.test 56e0ee83e90b57f5f3644cb7d1b36a067b7b8b19cdf0dedce45e5e13cf752f65
@@ -1653,8 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a5a1949259a26c3330d7b1dcf38a8dd2e9b2c3af8d52f937e8537df138c9a55a 81f62e99f27dedf3dc10fa1593c4cd9fc158680867206aee853e503a0cc8df71
-R fccb8367f74d1b5d089bf2ff62c634d9
-T +closed 81f62e99f27dedf3dc10fa1593c4cd9fc158680867206aee853e503a0cc8df71
+P 8525c30c1d6676e51b9d9c642450aae3722c129edafdbc39b8435fcfe09b1817
+R 6ec072e8cc5f89299e9fb58896934d7c
 U drh
-Z 70196bf0d2abbf2ab0f5b4a6633369ee
+Z 5fb925822a75ef5ce57cd890a3d631bf
diff --git a/manifest.uuid b/manifest.uuid
index b2d34d0136..5e4e028228 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8525c30c1d6676e51b9d9c642450aae3722c129edafdbc39b8435fcfe09b1817
\ No newline at end of file
+87ccdf9cbb9284553330683d4971be4f523ce922089aee6dffccfc18b3004263
\ No newline at end of file
diff --git a/test/fts3conf.test b/test/fts3conf.test
index 6766a95408..6ceef2c47e 100644
--- a/test/fts3conf.test
+++ b/test/fts3conf.test
@@ -136,47 +136,49 @@ do_execsql_test 2.2.2 { COMMIT }
 do_execsql_test 2.2.3 { SELECT * FROM t1 } {{a b c} {a b c}}
 fts3_integrity 2.2.4 db t1
 
-do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE t3 USING fts4;
-  REPLACE INTO t3(docid, content) VALUES (1, 'one two');
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
-} {X'0100000002000000'}
-
-do_execsql_test 3.2 {
-  REPLACE INTO t3(docid, content) VALUES (2, 'one two three four');
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'four'
-} {X'0200000003000000'}
-
-do_execsql_test 3.3 {
-  REPLACE INTO t3(docid, content) VALUES (1, 'one two three four five six');
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
-} {X'0200000005000000'}
-
-do_execsql_test 3.4 {
-  UPDATE OR REPLACE t3 SET docid = 2 WHERE docid=1;
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
-} {X'0100000006000000'}
-
-do_execsql_test 3.5 {
-  UPDATE OR REPLACE t3 SET docid = 3 WHERE docid=2;
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
-} {X'0100000006000000'}
-
-do_execsql_test 3.6 {
-  REPLACE INTO t3(docid, content) VALUES (3, 'one two');
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
-} {X'0100000002000000'}
-
-do_execsql_test 3.7 {
-  REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four');
-  REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four five six');
-  SELECT docid FROM t3;
-} {3 4 5}
-
-do_execsql_test 3.8 {
-  UPDATE OR REPLACE t3 SET docid = 5, content='three four' WHERE docid = 4;
-  SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
-} {X'0200000002000000'}
+if {$tcl_platform(byteOrder)=="littleEndian"} {
+  do_execsql_test 3.1 {
+    CREATE VIRTUAL TABLE t3 USING fts4;
+    REPLACE INTO t3(docid, content) VALUES (1, 'one two');
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
+  } {X'0100000002000000'}
+  
+  do_execsql_test 3.2 {
+    REPLACE INTO t3(docid, content) VALUES (2, 'one two three four');
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'four'
+  } {X'0200000003000000'}
+  
+  do_execsql_test 3.3 {
+    REPLACE INTO t3(docid, content) VALUES (1, 'one two three four five six');
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
+  } {X'0200000005000000'}
+  
+  do_execsql_test 3.4 {
+    UPDATE OR REPLACE t3 SET docid = 2 WHERE docid=1;
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
+  } {X'0100000006000000'}
+  
+  do_execsql_test 3.5 {
+    UPDATE OR REPLACE t3 SET docid = 3 WHERE docid=2;
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six'
+  } {X'0100000006000000'}
+  
+  do_execsql_test 3.6 {
+    REPLACE INTO t3(docid, content) VALUES (3, 'one two');
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
+  } {X'0100000002000000'}
+  
+  do_execsql_test 3.7 {
+    REPLACE INTO t3(docid, content) VALUES(NULL,'one two three four');
+    REPLACE INTO t3(docid, content) VALUES(NULL,'one two three four five six');
+    SELECT docid FROM t3;
+  } {3 4 5}
+  
+  do_execsql_test 3.8 {
+    UPDATE OR REPLACE t3 SET docid = 5, content='three four' WHERE docid = 4;
+    SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one'
+  } {X'0200000002000000'}
+}
 
 #-------------------------------------------------------------------------
 # Test that the xSavepoint is invoked correctly if the first write 

From bb6896226f3c833f1e3488fb6b20543e7644608b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Sep 2017 20:20:36 +0000
Subject: [PATCH 087/270] Test case update due to PRAGMA integrity_check
 enhancements in [8525c30c]. No changes to code.

FossilOrigin-Name: 43c6023bbf6b808ab4cfdbd1a63a516cbe2f1794c7787f8230632bae12e2ff59
---
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 test/corruptC.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 42cba1232c..5fe9f5a3e9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sFTS3\smatchinfo()\stests\sthat\sassume\sa\slittleEndian\splatform\swhen\nrunning\son\sbigEndian.
-D 2017-09-13T12:55:59.555
+C Test\scase\supdate\sdue\sto\sPRAGMA\sintegrity_check\senhancements\sin\s[8525c30c].\nNo\schanges\sto\scode.
+D 2017-09-13T20:20:36.313
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -676,7 +676,7 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516
 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85
 F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b
 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec
-F test/corruptC.test 46ec43bd90a02fd7b37ad8a7a949c55aa5717f89
+F test/corruptC.test 138ecb02188ed1a719b533d4a139568204039f72f00e07a8d30d920bd83122db
 F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040
 F test/corruptE.test 82ccf4f8f543fdbedd4aa42c709cb077f7374c62
 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8525c30c1d6676e51b9d9c642450aae3722c129edafdbc39b8435fcfe09b1817
-R 6ec072e8cc5f89299e9fb58896934d7c
+P 87ccdf9cbb9284553330683d4971be4f523ce922089aee6dffccfc18b3004263
+R 50570f8e00c8cd0ca9dde4dca24e8802
 U drh
-Z 5fb925822a75ef5ce57cd890a3d631bf
+Z 86679bd0091a9a37518dba2a956b32cd
diff --git a/manifest.uuid b/manifest.uuid
index 5e4e028228..66c7d2c062 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-87ccdf9cbb9284553330683d4971be4f523ce922089aee6dffccfc18b3004263
\ No newline at end of file
+43c6023bbf6b808ab4cfdbd1a63a516cbe2f1794c7787f8230632bae12e2ff59
\ No newline at end of file
diff --git a/test/corruptC.test b/test/corruptC.test
index f404e4fb5b..7c6ffb8102 100644
--- a/test/corruptC.test
+++ b/test/corruptC.test
@@ -164,7 +164,7 @@ do_test corruptC-2.5 {
   catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;}
   catchsql {PRAGMA integrity_check}
 } {0 {{*** in database main ***
-On tree page 4 cell 19: Extends off end of page}}}
+On tree page 4 cell 19: Extends off end of page} {database disk image is malformed}}}
 
 # {0 {{*** in database main ***
 # Corruption detected in cell 710 on page 4

From 263a8b660f401afb7fc1da94d34c37b896feebb0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 14 Sep 2017 02:36:27 +0000
Subject: [PATCH 088/270] Avoid an out-of-bounds read on a recovery attempt
 using a carefully crafted database and rollback journal with mismatched page
 sizes.  The test case for this is in TH3.

FossilOrigin-Name: 378afa16381a222aafa6009dbbbc92473a69683537f1c265694678b0595a42c8
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   |  8 ++++----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 5fe9f5a3e9..00998942b5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Test\scase\supdate\sdue\sto\sPRAGMA\sintegrity_check\senhancements\sin\s[8525c30c].\nNo\schanges\sto\scode.
-D 2017-09-13T20:20:36.313
+C Avoid\san\sout-of-bounds\sread\son\sa\srecovery\sattempt\susing\sa\scarefully\scrafted\ndatabase\sand\srollback\sjournal\swith\smismatched\spage\ssizes.\s\sThe\stest\scase\sfor\nthis\sis\sin\sTH3.
+D 2017-09-14T02:36:27.714
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -444,7 +444,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 967168bba88d2dc790ed9618bd4ba7bfe475b67b521ef6da305a6425c592928f
+F src/pager.c 2a523bf8ec77678b35fe56b43ac24045d2f97ad44d58c6a0894c131feda3eeff
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
@@ -1653,7 +1653,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 87ccdf9cbb9284553330683d4971be4f523ce922089aee6dffccfc18b3004263
-R 50570f8e00c8cd0ca9dde4dca24e8802
+P 43c6023bbf6b808ab4cfdbd1a63a516cbe2f1794c7787f8230632bae12e2ff59
+R 751623e94f5ae53084bb1caa4a774bfe
 U drh
-Z 86679bd0091a9a37518dba2a956b32cd
+Z 37756b1d45e36db238c0646514cfea00
diff --git a/manifest.uuid b/manifest.uuid
index 66c7d2c062..e1134062ff 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-43c6023bbf6b808ab4cfdbd1a63a516cbe2f1794c7787f8230632bae12e2ff59
\ No newline at end of file
+378afa16381a222aafa6009dbbbc92473a69683537f1c265694678b0595a42c8
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 4f3f75b6e7..87622f83a5 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -2844,12 +2844,13 @@ static int pager_playback(Pager *pPager, int isHot){
   ** pager_playback_one_page() call returns SQLITE_DONE or an IO error 
   ** occurs. 
   */
-  while( 1 ){
+  do{
     /* Read the next journal header from the journal file.  If there are
     ** not enough bytes left in the journal file for a complete header, or
     ** it is corrupted, then a process must have failed while writing it.
     ** This indicates nothing more needs to be rolled back.
     */
+    u32 savedPageSize = pPager->pageSize;
     rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg);
     if( rc!=SQLITE_OK ){ 
       if( rc==SQLITE_DONE ){
@@ -2931,9 +2932,8 @@ static int pager_playback(Pager *pPager, int isHot){
         }
       }
     }
-  }
-  /*NOTREACHED*/
-  assert( 0 );
+    rc = sqlite3PagerSetPagesize(pPager, &savedPageSize, -1);
+  }while( rc==SQLITE_OK );
 
 end_playback:
   /* Following a rollback, the database file should be back in its original

From 629ec14abc44aa8292923db945263f8d0f96440d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 14 Sep 2017 20:41:17 +0000
Subject: [PATCH 089/270] Avoid an out-of-bounds read that can be caused by a
 specially constructed journal file.

FossilOrigin-Name: cf5bf42cad6e019a38dc0a36ff1f53ada619eef5259e175c3554a16669e03202
---
 manifest           | 15 +++++------
 manifest.uuid      |  2 +-
 src/os_unix.c      | 25 ++++---------------
 test/mjournal.test | 62 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 76 insertions(+), 28 deletions(-)
 create mode 100644 test/mjournal.test

diff --git a/manifest b/manifest
index 00998942b5..7b324d258b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\san\sout-of-bounds\sread\son\sa\srecovery\sattempt\susing\sa\scarefully\scrafted\ndatabase\sand\srollback\sjournal\swith\smismatched\spage\ssizes.\s\sThe\stest\scase\sfor\nthis\sis\sin\sTH3.
-D 2017-09-14T02:36:27.714
+C Avoid\san\sout-of-bounds\sread\sthat\scan\sbe\scaused\sby\sa\sspecially\sconstructed\njournal\sfile.
+D 2017-09-14T20:41:17.170
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -441,7 +441,7 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
+F src/os_unix.c 0e365b4c95bd6f5a9d3cea650272eba46706bad2c833f1d0e7ac38521331ddba
 F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2a523bf8ec77678b35fe56b43ac24045d2f97ad44d58c6a0894c131feda3eeff
@@ -1046,6 +1046,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
+F test/mjournal.test 479076d56e89659cce2197ee0054df4a6578f43e10bdda9bdfcdb6eefaa02575
 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce
 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
@@ -1653,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 43c6023bbf6b808ab4cfdbd1a63a516cbe2f1794c7787f8230632bae12e2ff59
-R 751623e94f5ae53084bb1caa4a774bfe
-U drh
-Z 37756b1d45e36db238c0646514cfea00
+P 378afa16381a222aafa6009dbbbc92473a69683537f1c265694678b0595a42c8
+R c2746be599de5d7ee9f8e8da17342031
+U dan
+Z cb084e7d60ad5fef0134c6ef29bfedb3
diff --git a/manifest.uuid b/manifest.uuid
index e1134062ff..97f1e33d3b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-378afa16381a222aafa6009dbbbc92473a69683537f1c265694678b0595a42c8
\ No newline at end of file
+cf5bf42cad6e019a38dc0a36ff1f53ada619eef5259e175c3554a16669e03202
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 0d7e494147..3951989359 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5300,17 +5300,6 @@ static int fillInUnixFile(
 
   assert( pNew->pInode==NULL );
 
-  /* Usually the path zFilename should not be a relative pathname. The
-  ** exception is when opening the proxy "conch" file in builds that
-  ** include the special Apple locking styles.
-  */
-#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
-  assert( zFilename==0 || zFilename[0]=='/' 
-    || pVfs->pAppData==(void*)&autolockIoFinder );
-#else
-  assert( zFilename==0 || zFilename[0]=='/' );
-#endif
-
   /* No locking occurs in temporary files */
   assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 );
 
@@ -5673,16 +5662,11 @@ static int findCreateFileMode(
     */
     nDb = sqlite3Strlen30(zPath) - 1; 
     while( zPath[nDb]!='-' ){
-#ifndef SQLITE_ENABLE_8_3_NAMES
-      /* In the normal case (8+3 filenames disabled) the journal filename
-      ** is guaranteed to contain a '-' character. */
-      assert( nDb>0 );
-      assert( sqlite3Isalnum(zPath[nDb]) );
-#else
-      /* If 8+3 names are possible, then the journal file might not contain
-      ** a '-' character.  So check for that case and return early. */
+      /* In normal operation, the journal file name will always contain
+      ** a '-' character.  However in 8+3 filename mode, or if a corrupt
+      ** rollback journal specifies a master journal with a goofy name, then
+      ** the '-' might be missing. */
       if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK;
-#endif
       nDb--;
     }
     memcpy(zDb, zPath, nDb);
@@ -5968,6 +5952,7 @@ static int unixOpen(
   }
 #endif
   
+  assert( zPath==0 || zPath[0]=='/' || eType==SQLITE_OPEN_MASTER_JOURNAL );
   rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags);
 
 open_finished:
diff --git a/test/mjournal.test b/test/mjournal.test
new file mode 100644
index 0000000000..74d567e842
--- /dev/null
+++ b/test/mjournal.test
@@ -0,0 +1,62 @@
+# 2017 September 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix mjournal
+
+# Test that nothing bad happens if a journal file contains a pointer to
+# a master journal file that does not have a "-" in the name. At one point
+# this was causing a segfault on unix.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+}
+
+do_test 1.1 {
+  forcedelete test.db2journal test.db-journal
+
+  close [open test.db-journal w]
+  
+  hexio_write test.db-journal 0 746573742e6462326a6f75726e616c00
+  hexio_write test.db-journal 16 00000010
+  hexio_write test.db-journal 20 000005e1
+  hexio_write test.db-journal 24 d9d505f920a163d7
+
+  close [open test.db2journal w]
+  hexio_write test.db2journal 0 abcd
+} {2}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1;
+}
+
+do_test 1.3 {
+  forcedelete test0db2journal test.db-journal
+  close [open test.db-journal w]
+  hexio_write test.db-journal 0 74657374306462326a6f75726e616c00
+  hexio_write test.db-journal 16 00000010
+  hexio_write test.db-journal 20 000005e3
+  hexio_write test.db-journal 24 d9d505f920a163d7
+
+  close [open test0db2journal w]
+  hexio_write test0db2journal 0 abcd
+} {2}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1;
+}
+
+
+  
+finish_test

From 3ed0f1c9365742e778321bb309686342bed1b709 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 14 Sep 2017 21:12:07 +0000
Subject: [PATCH 090/270] Add another test case for the problem fixed by the
 previous commit.

FossilOrigin-Name: 1a7e0b61c8a6bdd3ed105b9bc8a3732668fd7a897d2ed16c99445498e3c87089
---
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 src/os_unix.c      |  4 +++-
 test/mjournal.test | 21 +++++++++++++++++++++
 4 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 7b324d258b..585d85f3a2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\san\sout-of-bounds\sread\sthat\scan\sbe\scaused\sby\sa\sspecially\sconstructed\njournal\sfile.
-D 2017-09-14T20:41:17.170
+C Add\sanother\stest\scase\sfor\sthe\sproblem\sfixed\sby\sthe\sprevious\scommit.
+D 2017-09-14T21:12:07.969
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -441,7 +441,7 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 0e365b4c95bd6f5a9d3cea650272eba46706bad2c833f1d0e7ac38521331ddba
+F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
 F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2a523bf8ec77678b35fe56b43ac24045d2f97ad44d58c6a0894c131feda3eeff
@@ -1046,7 +1046,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
-F test/mjournal.test 479076d56e89659cce2197ee0054df4a6578f43e10bdda9bdfcdb6eefaa02575
+F test/mjournal.test 68b749956f9a179e7e633a3958b48a5a905d28d30c7ec88f3f26dc6f220129db
 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce
 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 378afa16381a222aafa6009dbbbc92473a69683537f1c265694678b0595a42c8
-R c2746be599de5d7ee9f8e8da17342031
+P cf5bf42cad6e019a38dc0a36ff1f53ada619eef5259e175c3554a16669e03202
+R 8f7468b63be6b7c93b7232094070d3e5
 U dan
-Z cb084e7d60ad5fef0134c6ef29bfedb3
+Z 9fe988a13763bfb5ac0a7e60dbcb1440
diff --git a/manifest.uuid b/manifest.uuid
index 97f1e33d3b..784f3dd487 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf5bf42cad6e019a38dc0a36ff1f53ada619eef5259e175c3554a16669e03202
\ No newline at end of file
+1a7e0b61c8a6bdd3ed105b9bc8a3732668fd7a897d2ed16c99445498e3c87089
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 3951989359..4445104dd6 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5952,7 +5952,9 @@ static int unixOpen(
   }
 #endif
   
-  assert( zPath==0 || zPath[0]=='/' || eType==SQLITE_OPEN_MASTER_JOURNAL );
+  assert( zPath==0 || zPath[0]=='/' 
+      || eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL 
+  );
   rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags);
 
 open_finished:
diff --git a/test/mjournal.test b/test/mjournal.test
index 74d567e842..aab2c08b51 100644
--- a/test/mjournal.test
+++ b/test/mjournal.test
@@ -57,6 +57,27 @@ do_execsql_test 1.4 {
   SELECT * FROM t1;
 }
 
+# And now test that nothing bad happens if a master journal contains a
+# pointer to a journal file that does not have a "-" in the name. 
+#
+do_test 1.5 {
+  forcedelete test.db2-master test.db-journal test1
+  close [open test.db-journal w]
+  hexio_write test.db-journal 0 746573742e6462322d6d617374657200
+  hexio_write test.db-journal 16 00000010
+  hexio_write test.db-journal 20 0000059f
+  hexio_write test.db-journal 24 d9d505f920a163d7
+
+  close [open test.db2-master w]
+  hexio_write test.db2-master 0 746573743100
+
+  close [open test1 w]
+  hexio_write test1 0 abcd
+} {2}
+
+do_execsql_test 1.6 {
+  SELECT * FROM t1;
+}
 
   
 finish_test

From 154896e8d243bf895775707a1741950dc39567fd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Sep 2017 14:36:13 +0000
Subject: [PATCH 091/270] Fix a harmless comment typo.  No changes to code.

FossilOrigin-Name: f7f0bf1da03f7fc1647ef172d9cb71a2ac46f136d4dee8e3a24e39313a981eb5
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/where.c   |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 585d85f3a2..7005e5f16b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sanother\stest\scase\sfor\sthe\sproblem\sfixed\sby\sthe\sprevious\scommit.
-D 2017-09-14T21:12:07.969
+C Fix\sa\sharmless\scomment\stypo.\s\sNo\schanges\sto\scode.
+D 2017-09-15T14:36:13.956
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -539,7 +539,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c d97f97ca6017747b7361af7f168a640d667c41cd1d4e91c2c5b2bd7538815d07
+F src/where.c 21eea981920a13fd3c0ac3d6c128d0a34b22cbec064e4f0603375fe1ffe26ca6
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c 4953ca4e769c047d0a00a1ba9085849626b1f3a6e89f6befcf5c38fa0722acdd
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cf5bf42cad6e019a38dc0a36ff1f53ada619eef5259e175c3554a16669e03202
-R 8f7468b63be6b7c93b7232094070d3e5
-U dan
-Z 9fe988a13763bfb5ac0a7e60dbcb1440
+P 1a7e0b61c8a6bdd3ed105b9bc8a3732668fd7a897d2ed16c99445498e3c87089
+R 5c4f2e86b78512af5ee1ca6c5d751115
+U drh
+Z 3594cc4520cafe2c25c44febc34a036c
diff --git a/manifest.uuid b/manifest.uuid
index 784f3dd487..8f64ce1318 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1a7e0b61c8a6bdd3ed105b9bc8a3732668fd7a897d2ed16c99445498e3c87089
\ No newline at end of file
+f7f0bf1da03f7fc1647ef172d9cb71a2ac46f136d4dee8e3a24e39313a981eb5
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 7b806a7b6c..e5fad790b5 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4798,7 +4798,7 @@ WhereInfo *sqlite3WhereBegin(
       Index *pIx = pLoop->u.btree.pIndex;
       int iIndexCur;
       int op = OP_OpenRead;
-      /* iAuxArg is always set if to a positive value if ONEPASS is possible */
+      /* iAuxArg is always set to a positive value if ONEPASS is possible */
       assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
       if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
        && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0

From a19543fe703823a434d5d0702db9e45075958be1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Sep 2017 15:17:48 +0000
Subject: [PATCH 092/270] Optimization to the ExprList object to make it
 slightly smaller and faster.

FossilOrigin-Name: 4da49a95c0f07ed7790169e8833c3e2dacda504a3d997f567572020148abe30b
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/expr.c      | 21 ++++++++++++++++-----
 src/sqliteInt.h |  4 +++-
 4 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 7005e5f16b..092968091c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\scomment\stypo.\s\sNo\schanges\sto\scode.
-D 2017-09-15T14:36:13.956
+C Optimization\sto\sthe\sExprList\sobject\sto\smake\sit\sslightly\ssmaller\sand\sfaster.
+D 2017-09-15T15:17:48.630
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -410,7 +410,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 0f611840217016cf2c5e72f2eb8e412e48511bf740ae1fd5b58dc5e409c6e738
+F src/expr.c e44dda9df05faf96d340bbb68db3d1c47658576c13ac2fc3b660e0fe738d693e
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -463,7 +463,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 2272cd09b7f137bb2423af589d636b5b741037db9b329d359c1ef6556fac5a4c
+F src/sqliteInt.h 386e5b493c2e534c27366a8cdc20ed2f9767ec1eef63a4bad25a098c93aee473
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1a7e0b61c8a6bdd3ed105b9bc8a3732668fd7a897d2ed16c99445498e3c87089
-R 5c4f2e86b78512af5ee1ca6c5d751115
+P f7f0bf1da03f7fc1647ef172d9cb71a2ac46f136d4dee8e3a24e39313a981eb5
+R e5c2701ca337741bd503cd0a7cd02786
 U drh
-Z 3594cc4520cafe2c25c44febc34a036c
+Z 176d5b19d69323e742495e3d44c24aea
diff --git a/manifest.uuid b/manifest.uuid
index 8f64ce1318..ad88c5c5fd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f7f0bf1da03f7fc1647ef172d9cb71a2ac46f136d4dee8e3a24e39313a981eb5
\ No newline at end of file
+4da49a95c0f07ed7790169e8833c3e2dacda504a3d997f567572020148abe30b
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 0c3b608dee..38c799c6b3 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1305,7 +1305,11 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){
   pNew = sqlite3DbMallocRawNN(db, 
              sizeof(*pNew)+sizeof(pNew->a[0])*(p->nExpr-1) );
   if( pNew==0 ) return 0;
-  pNew->nAlloc = pNew->nExpr = p->nExpr;
+  pNew->nExpr = p->nExpr;
+  /* After being duplicated, the ExprList may not be expanded again using
+  ** Append() because Append() assumes that the number of slots in
+  ** ExprList.a[] is a power of 2 */
+  VVA_ONLY( pNew->bFixedSize = 1 );
   pItem = pNew->a;
   pOldItem = p->a;
   for(i=0; i<p->nExpr; i++, pItem++, pOldItem++){
@@ -1459,6 +1463,13 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
 ** Add a new element to the end of an expression list.  If pList is
 ** initially NULL, then create a new expression list.
 **
+** The pList argument must be either NULL or a pointer to an ExprList
+** obtained from a prior call to sqlite3ExprListAppend().  This routine
+** may not be used with an ExprList obtained from sqlite3ExprListDup().
+** Reason:  This routine assumes that the number of slots in pList->a[]
+** is a power of two.  That is true for sqlite3ExprListAppend() returns
+** but is not necessarily true from the return value of sqlite3ExprListDup().
+**
 ** If a memory allocation error occurs, the entire list is freed and
 ** NULL is returned.  If non-NULL is returned, then it is guaranteed
 ** that the new entry was successfully appended.
@@ -1471,22 +1482,22 @@ ExprList *sqlite3ExprListAppend(
   struct ExprList_item *pItem;
   sqlite3 *db = pParse->db;
   assert( db!=0 );
+  assert( pList==0 || pList->bFixedSize==0 );
   if( pList==0 ){
     pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) );
     if( pList==0 ){
       goto no_mem;
     }
     pList->nExpr = 0;
-    pList->nAlloc = 1;
-  }else if( pList->nExpr==pList->nAlloc ){
+    VVA_ONLY( pList->bFixedSize = 0 );
+  }else if( (pList->nExpr & (pList->nExpr-1))==0 ){
     ExprList *pNew;
     pNew = sqlite3DbRealloc(db, pList, 
-             sizeof(*pList)+(2*pList->nAlloc - 1)*sizeof(pList->a[0]));
+             sizeof(*pList)+(2*pList->nExpr - 1)*sizeof(pList->a[0]));
     if( pNew==0 ){
       goto no_mem;
     }
     pList = pNew;
-    pList->nAlloc *= 2;
   }
   pItem = &pList->a[pList->nExpr++];
   assert( offsetof(struct ExprList_item,zName)==sizeof(pItem->pExpr) );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 58d64ba2f2..4519a472a8 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2468,7 +2468,9 @@ struct Expr {
 */
 struct ExprList {
   int nExpr;             /* Number of expressions on the list */
-  int nAlloc;            /* Number of a[] slots allocated */
+#ifdef SQLITE_DEBUG
+  u8 bFixedSize;         /* May not be expanded using sqlite3ExprListAppend() */
+#endif
   struct ExprList_item { /* For each expression in the list */
     Expr *pExpr;            /* The parse tree for this expression */
     char *zName;            /* Token associated with this expression */

From 3df6c3b1c5fad0204902846b27c4b5ccffeda57f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Sep 2017 15:38:01 +0000
Subject: [PATCH 093/270] Improved the header-comment documentation on
 sqlite3ExprCodeExprList(). No changes to code.

FossilOrigin-Name: 5dc3ecb5f67968545fb35ceed61ad625e069c4e744c5c3ebaea65e2fee347a5f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/expr.c    |  6 +++++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 092968091c..ee10fd079b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Optimization\sto\sthe\sExprList\sobject\sto\smake\sit\sslightly\ssmaller\sand\sfaster.
-D 2017-09-15T15:17:48.630
+C Improved\sthe\sheader-comment\sdocumentation\son\ssqlite3ExprCodeExprList().\nNo\schanges\sto\scode.
+D 2017-09-15T15:38:01.722
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -410,7 +410,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c e44dda9df05faf96d340bbb68db3d1c47658576c13ac2fc3b660e0fe738d693e
+F src/expr.c 995a0801cc3ed8491cccbd6851b5d1f2461a67363e03cff377dd2a275b8a894d
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f7f0bf1da03f7fc1647ef172d9cb71a2ac46f136d4dee8e3a24e39313a981eb5
-R e5c2701ca337741bd503cd0a7cd02786
+P 4da49a95c0f07ed7790169e8833c3e2dacda504a3d997f567572020148abe30b
+R bec2d8e8b1fdf47c35ba1679c00b4d84
 U drh
-Z 176d5b19d69323e742495e3d44c24aea
+Z 9190f931e57ef2b5b105e255f9cf8a27
diff --git a/manifest.uuid b/manifest.uuid
index ad88c5c5fd..01d19c788e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4da49a95c0f07ed7790169e8833c3e2dacda504a3d997f567572020148abe30b
\ No newline at end of file
+5dc3ecb5f67968545fb35ceed61ad625e069c4e744c5c3ebaea65e2fee347a5f
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 38c799c6b3..405103f21b 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -4252,7 +4252,9 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){
 ** Generate code that pushes the value of every element of the given
 ** expression list into a sequence of registers beginning at target.
 **
-** Return the number of elements evaluated.
+** Return the number of elements evaluated.  The number returned will
+** usually be pList->nExpr but might be reduced if SQLITE_ECEL_OMITREF
+** is defined.
 **
 ** The SQLITE_ECEL_DUP flag prevents the arguments from being
 ** filled using OP_SCopy.  OP_Copy must be used instead.
@@ -4263,6 +4265,8 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){
 ** The SQLITE_ECEL_REF flag means that expressions in the list with
 ** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored
 ** in registers at srcReg, and so the value can be copied from there.
+** If SQLITE_ECEL_OMITREF is also set, then the values with u.x.iOrderByCol>0
+** are simply omitted rather than being copied from srcReg.
 */
 int sqlite3ExprCodeExprList(
   Parse *pParse,     /* Parsing context */

From 2def2f7e0ace209fb9966d6322d5750a0efb0c1e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Sep 2017 17:40:34 +0000
Subject: [PATCH 094/270] Remove an unnecessary parameter from
 selectInnerLoop().

FossilOrigin-Name: dd95887f82739d05585edfb70913fdaaab154de46c4d3113dd32d283767fb6fc
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 36 ++++++++++++++++++------------------
 3 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index ee10fd079b..7b71d5a060 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sthe\sheader-comment\sdocumentation\son\ssqlite3ExprCodeExprList().\nNo\schanges\sto\scode.
-D 2017-09-15T15:38:01.722
+C Remove\san\sunnecessary\sparameter\sfrom\sselectInnerLoop().
+D 2017-09-15T17:40:34.507
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -457,7 +457,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
+F src/select.c 5945cf1a5426c279c9f3c6ad16d24c3dc55349b69afba4f35cf3fe115418b122
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4da49a95c0f07ed7790169e8833c3e2dacda504a3d997f567572020148abe30b
-R bec2d8e8b1fdf47c35ba1679c00b4d84
+P 5dc3ecb5f67968545fb35ceed61ad625e069c4e744c5c3ebaea65e2fee347a5f
+R bedf4233ae6bc36b349315bc4854b90d
 U drh
-Z 9190f931e57ef2b5b105e255f9cf8a27
+Z 884e6c4ed9b99d145e55ac32471e70b4
diff --git a/manifest.uuid b/manifest.uuid
index 01d19c788e..4833c67174 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5dc3ecb5f67968545fb35ceed61ad625e069c4e744c5c3ebaea65e2fee347a5f
\ No newline at end of file
+dd95887f82739d05585edfb70913fdaaab154de46c4d3113dd32d283767fb6fc
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index aedbcc4b3a..53e978eac2 100644
--- a/src/select.c
+++ b/src/select.c
@@ -664,16 +664,15 @@ static void codeDistinct(
 ** This routine generates the code for the inside of the inner loop
 ** of a SELECT.
 **
-** If srcTab is negative, then the pEList expressions
+** If srcTab is negative, then the p->pEList expressions
 ** are evaluated in order to get the data for this row.  If srcTab is
-** zero or more, then data is pulled from srcTab and pEList is used only 
+** zero or more, then data is pulled from srcTab and p->pEList is used only 
 ** to get the number of columns and the collation sequence for each column.
 */
 static void selectInnerLoop(
   Parse *pParse,          /* The parser context */
   Select *p,              /* The complete select statement being coded */
-  ExprList *pEList,       /* List of values being extracted */
-  int srcTab,             /* Pull data from this table */
+  int srcTab,             /* Pull data from this table if non-negative */
   SortCtx *pSort,         /* If not NULL, info on how to process ORDER BY */
   DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */
   SelectDest *pDest,      /* How to dispose of the results */
@@ -697,7 +696,7 @@ static void selectInnerLoop(
   int regOrig;                /* Start of memory holding full result (or 0) */
 
   assert( v );
-  assert( pEList!=0 );
+  assert( p->pEList!=0 );
   hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP;
   if( pSort && pSort->pOrderBy==0 ) pSort = 0;
   if( pSort==0 && !hasDistinct ){
@@ -707,7 +706,7 @@ static void selectInnerLoop(
 
   /* Pull the requested columns.
   */
-  nResultCol = pEList->nExpr;
+  nResultCol = p->pEList->nExpr;
 
   if( pDest->iSdst==0 ){
     if( pSort ){
@@ -730,7 +729,7 @@ static void selectInnerLoop(
   if( srcTab>=0 ){
     for(i=0; i<nResultCol; i++){
       sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
-      VdbeComment((v, "%s", pEList->a[i].zName));
+      VdbeComment((v, "%s", p->pEList->a[i].zName));
     }
   }else if( eDest!=SRT_Exists ){
     /* If the destination is an EXISTS(...) expression, the actual
@@ -743,24 +742,24 @@ static void selectInnerLoop(
       ecelFlags = 0;
     }
     if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){
-      /* For each expression in pEList that is a copy of an expression in
+      /* For each expression in p->pEList that is a copy of an expression in
       ** the ORDER BY clause (pSort->pOrderBy), set the associated 
       ** iOrderByCol value to one more than the index of the ORDER BY 
       ** expression within the sort-key that pushOntoSorter() will generate.
-      ** This allows the pEList field to be omitted from the sorted record,
+      ** This allows the p->pEList field to be omitted from the sorted record,
       ** saving space and CPU cycles.  */
       ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
       for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
         int j;
         if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
-          pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
+          p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
         }
       }
       regOrig = 0;
       assert( eDest==SRT_Set || eDest==SRT_Mem 
            || eDest==SRT_Coroutine || eDest==SRT_Output );
     }
-    nResultCol = sqlite3ExprCodeExprList(pParse,pEList,regResult,0,ecelFlags);
+    nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,0,ecelFlags);
   }
 
   /* If the DISTINCT keyword was present on the SELECT statement
@@ -792,7 +791,7 @@ static void selectInnerLoop(
 
         iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
         for(i=0; i<nResultCol; i++){
-          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr);
+          CollSeq *pColl = sqlite3ExprCollSeq(pParse, p->pEList->a[i].pExpr);
           if( i<nResultCol-1 ){
             sqlite3VdbeAddOp3(v, OP_Ne, regResult+i, iJump, regPrev+i);
             VdbeCoverage(v);
@@ -2147,7 +2146,7 @@ static void generateWithRecursiveQuery(
   /* Output the single row in Current */
   addrCont = sqlite3VdbeMakeLabel(v);
   codeOffset(v, regOffset, addrCont);
-  selectInnerLoop(pParse, p, p->pEList, iCurrent,
+  selectInnerLoop(pParse, p, iCurrent,
       0, 0, pDest, addrCont, addrBreak);
   if( regLimit ){
     sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak);
@@ -2466,7 +2465,7 @@ static int multiSelect(
         computeLimitRegisters(pParse, p, iBreak);
         sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v);
         iStart = sqlite3VdbeCurrentAddr(v);
-        selectInnerLoop(pParse, p, p->pEList, unionTab,
+        selectInnerLoop(pParse, p, unionTab,
                         0, 0, &dest, iCont, iBreak);
         sqlite3VdbeResolveLabel(v, iCont);
         sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v);
@@ -2539,7 +2538,7 @@ static int multiSelect(
       iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1);
       sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v);
       sqlite3ReleaseTempReg(pParse, r1);
-      selectInnerLoop(pParse, p, p->pEList, tab1,
+      selectInnerLoop(pParse, p, tab1,
                       0, 0, &dest, iCont, iBreak);
       sqlite3VdbeResolveLabel(v, iCont);
       sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v);
@@ -5545,7 +5544,8 @@ int sqlite3Select(
     }
 
     /* Use the standard inner loop. */
-    selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest,
+    assert( p->pEList==pEList );
+    selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest,
                     sqlite3WhereContinueLabel(pWInfo),
                     sqlite3WhereBreakLabel(pWInfo));
 
@@ -5848,7 +5848,7 @@ int sqlite3Select(
       sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
       finalizeAggFunctions(pParse, &sAggInfo);
       sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL);
-      selectInnerLoop(pParse, p, p->pEList, -1, &sSort,
+      selectInnerLoop(pParse, p, -1, &sSort,
                       &sDistinct, pDest,
                       addrOutputRow+1, addrSetAbort);
       sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
@@ -5992,7 +5992,7 @@ int sqlite3Select(
 
       sSort.pOrderBy = 0;
       sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL);
-      selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, 
+      selectInnerLoop(pParse, p, -1, 0, 0, 
                       pDest, addrEnd, addrEnd);
       sqlite3ExprListDelete(db, pDel);
     }

From 559656196bed206a08c9d07ae5610f1ea95243de Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 16 Sep 2017 20:58:41 +0000
Subject: [PATCH 095/270] Simplify the sqlite3VdbeGet() routine.  Smaller,
 faster, and easier to maintain.

FossilOrigin-Name: 5dbb255a9377a6c2619fbac519f18caa4d8cb23257dfba0ffb9e36dd9dc16627
---
 manifest       | 16 ++++++++--------
 manifest.uuid  |  2 +-
 src/select.c   | 13 +++++--------
 src/vdbeaux.c  |  2 ++
 src/vdbeblob.c |  5 +++--
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index 7b71d5a060..4346a87a24 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\sparameter\sfrom\sselectInnerLoop().
-D 2017-09-15T17:40:34.507
+C Simplify\sthe\ssqlite3VdbeGet()\sroutine.\s\sSmaller,\sfaster,\sand\seasier\sto\smaintain.
+D 2017-09-16T20:58:41.545
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -457,7 +457,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 5945cf1a5426c279c9f3c6ad16d24c3dc55349b69afba4f35cf3fe115418b122
+F src/select.c 6ea1cb2f0ab80a5bbaa603c872658c0237d4997d24a455e9b025d74ea08a4615
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -529,8 +529,8 @@ F src/vdbe.c 0a0ef7f0759ee4c3ed5ff0c6e9641216d4b31ebb0fea9a7b5a9c4a86ce875ff3
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 41d90002d774e234f95d1298fc70d25e3420e8d9ebd65aa270a8b771c5525790
-F src/vdbeblob.c db3cf91060f6f4b2f1358a4200e844697990752177784c7c95da00b7ac9f1c7b
+F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
+F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5dc3ecb5f67968545fb35ceed61ad625e069c4e744c5c3ebaea65e2fee347a5f
-R bedf4233ae6bc36b349315bc4854b90d
+P dd95887f82739d05585edfb70913fdaaab154de46c4d3113dd32d283767fb6fc
+R b8f3b2c060c77e9294810570628283d5
 U drh
-Z 884e6c4ed9b99d145e55ac32471e70b4
+Z 0a435a49b3c57e3c1d1922c1afe31527
diff --git a/manifest.uuid b/manifest.uuid
index 4833c67174..da58c0d08b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dd95887f82739d05585edfb70913fdaaab154de46c4d3113dd32d283767fb6fc
\ No newline at end of file
+5dbb255a9377a6c2619fbac519f18caa4d8cb23257dfba0ffb9e36dd9dc16627
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 53e978eac2..2b4ec33815 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1861,19 +1861,16 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
 ** Get a VDBE for the given parser context.  Create a new one if necessary.
 ** If an error occurs, return NULL and leave a message in pParse.
 */
-static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){
-  Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse);
-  if( v ) sqlite3VdbeAddOp2(v, OP_Init, 0, 1);
+Vdbe *sqlite3GetVdbe(Parse *pParse){
+  if( pParse->pVdbe ){
+    return pParse->pVdbe;
+  }
   if( pParse->pToplevel==0
    && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst)
   ){
     pParse->okConstFactor = 1;
   }
-  return v;
-}
-Vdbe *sqlite3GetVdbe(Parse *pParse){
-  Vdbe *v = pParse->pVdbe;
-  return v ? v : allocVdbe(pParse);
+  return sqlite3VdbeCreate(pParse);
 }
 
 
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index e5465d16e7..d426e86732 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -33,10 +33,12 @@ Vdbe *sqlite3VdbeCreate(Parse *pParse){
   db->pVdbe = p;
   p->magic = VDBE_MAGIC_INIT;
   p->pParse = pParse;
+  pParse->pVdbe = p;
   assert( pParse->aLabel==0 );
   assert( pParse->nLabel==0 );
   assert( pParse->nOpAlloc==0 );
   assert( pParse->szOpAlloc==0 );
+  sqlite3VdbeAddOp2(p, OP_Init, 0, 1);
   return p;
 }
 
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index 8d75f8ab01..16a2c0f36f 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -275,7 +275,8 @@ int sqlite3_blob_open(
       sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, 
                            pTab->pSchema->schema_cookie,
                            pTab->pSchema->iGeneration);
-      sqlite3VdbeChangeP5(v, 1);     
+      sqlite3VdbeChangeP5(v, 1);
+      assert( sqlite3VdbeCurrentAddr(v)==2 || db->mallocFailed );
       aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn);
 
       /* Make sure a mutex is held on the table to be accessed */
@@ -290,7 +291,7 @@ int sqlite3_blob_open(
         aOp[0].p1 = iDb;
         aOp[0].p2 = pTab->tnum;
         aOp[0].p3 = wrFlag;
-        sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
+        sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT);
       }
       if( db->mallocFailed==0 ){
 #endif

From 97258194a2eb054822fbb8ff6cf4d285458f5513 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 17 Sep 2017 19:45:28 +0000
Subject: [PATCH 096/270] Do not make the assumption (as check-in
 [4da49a95c0f07] incorrectly did) that the ExprList returned by
 sqlite3ExprListDup() would never be passed into sqlite3ExprListAppend(). 
 Include a new test case that shows this sometimes does happen.

FossilOrigin-Name: 29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/expr.c      |  9 +--------
 src/sqliteInt.h |  3 ---
 test/misc1.test | 10 ++++++++++
 5 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 4346a87a24..3427b106ce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplify\sthe\ssqlite3VdbeGet()\sroutine.\s\sSmaller,\sfaster,\sand\seasier\sto\smaintain.
-D 2017-09-16T20:58:41.545
+C Do\snot\smake\sthe\sassumption\s(as\scheck-in\s[4da49a95c0f07]\sincorrectly\sdid)\sthat\nthe\sExprList\sreturned\sby\ssqlite3ExprListDup()\swould\snever\sbe\spassed\sinto\nsqlite3ExprListAppend().\s\sInclude\sa\snew\stest\scase\sthat\sshows\sthis\ssometimes\ndoes\shappen.
+D 2017-09-17T19:45:28.163
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -410,7 +410,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 995a0801cc3ed8491cccbd6851b5d1f2461a67363e03cff377dd2a275b8a894d
+F src/expr.c 38a225057f5b7d6a2bc403d7a5d3a67f59ee57b73fe7c546221e3c53202a3731
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -463,7 +463,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 386e5b493c2e534c27366a8cdc20ed2f9767ec1eef63a4bad25a098c93aee473
+F src/sqliteInt.h 12aa1f626b3209ffa6a50d9d1e6b4235abd33273a0fcbfeedb66f573a68932b9
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1037,7 +1037,7 @@ F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
-F test/misc1.test 51ec3f56a2a7965de226964cff856695e743186826561536647f1e8b7d1d0eb3
+F test/misc1.test 76737c259537586355f45e2a1e121b6e91b5476c4604ad5c53d1abfcb3acf786
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dd95887f82739d05585edfb70913fdaaab154de46c4d3113dd32d283767fb6fc
-R b8f3b2c060c77e9294810570628283d5
+P 5dbb255a9377a6c2619fbac519f18caa4d8cb23257dfba0ffb9e36dd9dc16627
+R 42a326f02605d68acfe72daa45650e4b
 U drh
-Z 0a435a49b3c57e3c1d1922c1afe31527
+Z c92fbfe48a95c4de010b904ec47e6dc7
diff --git a/manifest.uuid b/manifest.uuid
index da58c0d08b..df51a44c4c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5dbb255a9377a6c2619fbac519f18caa4d8cb23257dfba0ffb9e36dd9dc16627
\ No newline at end of file
+29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 405103f21b..f3326d02e7 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1302,14 +1302,9 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){
   Expr *pPriorSelectCol = 0;
   assert( db!=0 );
   if( p==0 ) return 0;
-  pNew = sqlite3DbMallocRawNN(db, 
-             sizeof(*pNew)+sizeof(pNew->a[0])*(p->nExpr-1) );
+  pNew = sqlite3DbMallocRawNN(db, sqlite3DbMallocSize(db, p));
   if( pNew==0 ) return 0;
   pNew->nExpr = p->nExpr;
-  /* After being duplicated, the ExprList may not be expanded again using
-  ** Append() because Append() assumes that the number of slots in
-  ** ExprList.a[] is a power of 2 */
-  VVA_ONLY( pNew->bFixedSize = 1 );
   pItem = pNew->a;
   pOldItem = p->a;
   for(i=0; i<p->nExpr; i++, pItem++, pOldItem++){
@@ -1482,14 +1477,12 @@ ExprList *sqlite3ExprListAppend(
   struct ExprList_item *pItem;
   sqlite3 *db = pParse->db;
   assert( db!=0 );
-  assert( pList==0 || pList->bFixedSize==0 );
   if( pList==0 ){
     pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) );
     if( pList==0 ){
       goto no_mem;
     }
     pList->nExpr = 0;
-    VVA_ONLY( pList->bFixedSize = 0 );
   }else if( (pList->nExpr & (pList->nExpr-1))==0 ){
     ExprList *pNew;
     pNew = sqlite3DbRealloc(db, pList, 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 4519a472a8..14514007c4 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2468,9 +2468,6 @@ struct Expr {
 */
 struct ExprList {
   int nExpr;             /* Number of expressions on the list */
-#ifdef SQLITE_DEBUG
-  u8 bFixedSize;         /* May not be expanded using sqlite3ExprListAppend() */
-#endif
   struct ExprList_item { /* For each expression in the list */
     Expr *pExpr;            /* The parse tree for this expression */
     char *zName;            /* Token associated with this expression */
diff --git a/test/misc1.test b/test/misc1.test
index be64a8f7a9..2acfa5c2dd 100644
--- a/test/misc1.test
+++ b/test/misc1.test
@@ -711,5 +711,15 @@ SELECT-1 UNION  SELECT 5 UNION SELECT 0 UNION SElECT*from(SELECT-5) UNION SELECT
 $group,:conc ap0,1)fro,(select"",:PBAG,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d, foreign_keysc,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,a,b,d,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,bb,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,a,b,d,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,MAato_aecSELEC,+?b," "O,"i","a",""b  ,5 ))KEY)SELECT*FROM((k()reaC,k,K) eA,k '' )t ,K  M);
 } {1 {'k' is not a function}}
 
+# 2017-09-17
+#
+# Sometimes sqlite3ExprListAppend() can be invoked on an ExprList that
+# was obtained from sqlite3ExprListDup().
+#
+do_execsql_test misc1-26.0 {
+  DROP TABLE IF EXISTS abc;
+  CREATE TABLE abc(a, b, c);
+  SELECT randomblob(min(max(coalesce(EXISTS (SELECT 1 FROM ( SELECT (SELECT 2147483647) NOT IN (SELECT 2147483649 UNION ALL SELECT DISTINCT -1) IN (SELECT 2147483649), 'fault', (SELECT ALL -1 INTERSECT SELECT 'experiments') IN (SELECT ALL 56.1 ORDER BY 'experiments' DESC) FROM (SELECT DISTINCT 2147483648, 'hardware' UNION ALL SELECT -2147483648, 'experiments' ORDER BY 2147483648 LIMIT 1 OFFSET 123456789.1234567899) GROUP BY (SELECT ALL 0 INTERSECT SELECT 'in') IN (SELECT DISTINCT 'experiments' ORDER BY zeroblob(1000) LIMIT 56.1 OFFSET -456) HAVING EXISTS (SELECT 'fault' EXCEPT    SELECT DISTINCT 56.1) UNION SELECT 'The', 'The', 2147483649 UNION ALL SELECT DISTINCT 'hardware', 'first', 'experiments' ORDER BY 'hardware' LIMIT 123456789.1234567899 OFFSET -2147483647)) NOT IN (SELECT (SELECT DISTINCT (SELECT 'The') FROM abc ORDER BY EXISTS (SELECT -1 INTERSECT SELECT ALL NULL) ASC) IN (SELECT DISTINCT EXISTS (SELECT ALL 123456789.1234567899 ORDER BY 1 ASC, NULL DESC) FROM sqlite_master INTERSECT SELECT 456)), (SELECT ALL 'injection' UNION ALL SELECT ALL (SELECT DISTINCT 'first' UNION     SELECT DISTINCT 'The') FROM (SELECT 456, 'in', 2147483649))),1), 500)), 'first', EXISTS (SELECT DISTINCT 456 FROM abc ORDER BY 'experiments' DESC) FROM abc;
+} {}
 
 finish_test

From 2acd24d90c7b47436bc744f07ae2d714abd13e62 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 18 Sep 2017 00:18:31 +0000
Subject: [PATCH 097/270] Fix the CSV virtual table extension so that it works
 when the default character is unsigned.

FossilOrigin-Name: 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
---
 ext/misc/csv.c       |  4 ++--
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 test/releasetest.tcl |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/ext/misc/csv.c b/ext/misc/csv.c
index 1eafc3a414..c634ab3a4d 100644
--- a/ext/misc/csv.c
+++ b/ext/misc/csv.c
@@ -78,7 +78,7 @@ struct CsvReader {
   int nAlloc;            /* Space allocated for z[] */
   int nLine;             /* Current line number */
   int bNotFirst;         /* True if prior text has been seen */
-  char cTerm;            /* Character that terminated the most recent field */
+  int cTerm;             /* Character that terminated the most recent field */
   size_t iIn;            /* Next unread character in the input buffer */
   size_t nIn;            /* Number of characters in the input buffer */
   char *zIn;             /* The input buffer */
@@ -166,7 +166,7 @@ static int csv_getc(CsvReader *p){
     if( p->in!=0 ) return csv_getc_refill(p);
     return EOF;
   }
-  return p->zIn[p->iIn++];
+  return ((unsigned char*)p->zIn)[p->iIn++];
 }
 
 /* Increase the size of p->z and append character c to the end. 
diff --git a/manifest b/manifest
index 3427b106ce..3c256819da 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\smake\sthe\sassumption\s(as\scheck-in\s[4da49a95c0f07]\sincorrectly\sdid)\sthat\nthe\sExprList\sreturned\sby\ssqlite3ExprListDup()\swould\snever\sbe\spassed\sinto\nsqlite3ExprListAppend().\s\sInclude\sa\snew\stest\scase\sthat\sshows\sthis\ssometimes\ndoes\shappen.
-D 2017-09-17T19:45:28.163
+C Fix\sthe\sCSV\svirtual\stable\sextension\sso\sthat\sit\sworks\swhen\sthe\sdefault\scharacter\nis\sunsigned.
+D 2017-09-18T00:18:31.200
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -262,7 +262,7 @@ F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
-F ext/misc/csv.c b10ea114cf29f446e384064ae61e01373b431f6f172896f41ca0ba3507709003
+F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c b1aa06c0f1dac277695d4529e5e976c65ab5678dcbb53a0304deaa8adc44b332
@@ -1121,7 +1121,7 @@ F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
-F test/releasetest.tcl 7bb585433ce7fb2a2c255ae4b5e24f1bc27fe177ec1120f886cc4852f48f5ee9 x
+F test/releasetest.tcl 22bd6be9f227a1ad01ebfaa8ed92400b5eaaff1ecc1e14e269eec2631c7f43ab x
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test f580934279800d480a19176c6b44909df31ce7ad45267ea475a541daa522f3d3
 F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5dbb255a9377a6c2619fbac519f18caa4d8cb23257dfba0ffb9e36dd9dc16627
-R 42a326f02605d68acfe72daa45650e4b
+P 29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
+R ccebe3b4b57d7dc8c5482ee2b9f63107
 U drh
-Z c92fbfe48a95c4de010b904ec47e6dc7
+Z a535a0ac2d4006e12331e0812c689362
diff --git a/manifest.uuid b/manifest.uuid
index df51a44c4c..811612d113 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
\ No newline at end of file
+42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
\ No newline at end of file
diff --git a/test/releasetest.tcl b/test/releasetest.tcl
index 614ce35458..d5f758274d 100755
--- a/test/releasetest.tcl
+++ b/test/releasetest.tcl
@@ -114,7 +114,7 @@ array set ::Configs [strip_comments {
   }
   "Debug-One" {
     --disable-shared
-    -O2
+    -O2 -funsigned-char
     -DSQLITE_DEBUG=1
     -DSQLITE_MEMDEBUG=1
     -DSQLITE_MUTEX_NOOP=1

From edea4a7cc9929b1aeea17385c2c84449deb263f6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 18 Sep 2017 09:40:46 +0000
Subject: [PATCH 098/270] The out-of-bounds read on recovery fix of check-in
 [378afa16381a222a] caused problems for some corner-case error conditions. 
 This alternative fix appears to work better.

FossilOrigin-Name: 74f399d81fe250e09fde730ac0c17fe8b2b776ed32a3f576adaad56090f2b7fa
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   | 12 ++++++++----
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 3c256819da..2f300cfbdd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sCSV\svirtual\stable\sextension\sso\sthat\sit\sworks\swhen\sthe\sdefault\scharacter\nis\sunsigned.
-D 2017-09-18T00:18:31.200
+C The\sout-of-bounds\sread\son\srecovery\sfix\sof\scheck-in\s[378afa16381a222a]\scaused\nproblems\sfor\ssome\scorner-case\serror\sconditions.\s\sThis\salternative\sfix\sappears\nto\swork\sbetter.
+D 2017-09-18T09:40:46.165
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -444,7 +444,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
 F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 2a523bf8ec77678b35fe56b43ac24045d2f97ad44d58c6a0894c131feda3eeff
+F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
-R ccebe3b4b57d7dc8c5482ee2b9f63107
+P 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
+R 243e075c375b949edc613e149835756a
 U drh
-Z a535a0ac2d4006e12331e0812c689362
+Z d7a1bb52fe8a802fdbcc937d885ced35
diff --git a/manifest.uuid b/manifest.uuid
index 811612d113..2c0d1ff6fe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
\ No newline at end of file
+74f399d81fe250e09fde730ac0c17fe8b2b776ed32a3f576adaad56090f2b7fa
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 87622f83a5..a43614cdb4 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -2807,6 +2807,7 @@ static int pager_playback(Pager *pPager, int isHot){
   char *zMaster = 0;       /* Name of master journal file if any */
   int needPagerReset;      /* True to reset page prior to first page rollback */
   int nPlayback = 0;       /* Total number of pages restored from journal */
+  u32 savedPageSize = pPager->pageSize;
 
   /* Figure out how many records are in the journal.  Abort early if
   ** the journal is empty.
@@ -2844,13 +2845,12 @@ static int pager_playback(Pager *pPager, int isHot){
   ** pager_playback_one_page() call returns SQLITE_DONE or an IO error 
   ** occurs. 
   */
-  do{
+  while( 1 ){
     /* Read the next journal header from the journal file.  If there are
     ** not enough bytes left in the journal file for a complete header, or
     ** it is corrupted, then a process must have failed while writing it.
     ** This indicates nothing more needs to be rolled back.
     */
-    u32 savedPageSize = pPager->pageSize;
     rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg);
     if( rc!=SQLITE_OK ){ 
       if( rc==SQLITE_DONE ){
@@ -2932,10 +2932,14 @@ static int pager_playback(Pager *pPager, int isHot){
         }
       }
     }
-    rc = sqlite3PagerSetPagesize(pPager, &savedPageSize, -1);
-  }while( rc==SQLITE_OK );
+  }
+  /*NOTREACHED*/
+  assert( 0 );
 
 end_playback:
+  if( rc==SQLITE_OK ){
+    rc = sqlite3PagerSetPagesize(pPager, &savedPageSize, -1);
+  }
   /* Following a rollback, the database file should be back in its original
   ** state prior to the start of the transaction, so invoke the
   ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the

From 63347e7df8e78aa7d01cc94d16d45d1d1997d4ef Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 18 Sep 2017 13:16:10 +0000
Subject: [PATCH 099/270] Minor simplification of error message generation
 during compound query code generation.

FossilOrigin-Name: a944719314e0ac2f1954b65668815769eba3ab3e39a74666293b8dea52a184b2
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 12 +++---------
 3 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 2f300cfbdd..9b291427f1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sout-of-bounds\sread\son\srecovery\sfix\sof\scheck-in\s[378afa16381a222a]\scaused\nproblems\sfor\ssome\scorner-case\serror\sconditions.\s\sThis\salternative\sfix\sappears\nto\swork\sbetter.
-D 2017-09-18T09:40:46.165
+C Minor\ssimplification\sof\serror\smessage\sgeneration\sduring\scompound\squery\ncode\sgeneration.
+D 2017-09-18T13:16:10.771
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
@@ -457,7 +457,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 6ea1cb2f0ab80a5bbaa603c872658c0237d4997d24a455e9b025d74ea08a4615
+F src/select.c 9fa0db382f43217e207a145b8c6cec26e85cd1a42a8428ee8b3df5870dfea0f4
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1654,7 +1654,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
-R 243e075c375b949edc613e149835756a
+P 74f399d81fe250e09fde730ac0c17fe8b2b776ed32a3f576adaad56090f2b7fa
+R 1904950103dccd08d21013a5f29bd183
 U drh
-Z d7a1bb52fe8a802fdbcc937d885ced35
+Z b85cfdf5b76cc24c75500946d403051a
diff --git a/manifest.uuid b/manifest.uuid
index 2c0d1ff6fe..ef3ba538cd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-74f399d81fe250e09fde730ac0c17fe8b2b776ed32a3f576adaad56090f2b7fa
\ No newline at end of file
+a944719314e0ac2f1954b65668815769eba3ab3e39a74666293b8dea52a184b2
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 2b4ec33815..971e7f5d4e 100644
--- a/src/select.c
+++ b/src/select.c
@@ -2281,15 +2281,9 @@ static int multiSelect(
   db = pParse->db;
   pPrior = p->pPrior;
   dest = *pDest;
-  if( pPrior->pOrderBy ){
-    sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before",
-      selectOpName(p->op));
-    rc = 1;
-    goto multi_select_end;
-  }
-  if( pPrior->pLimit ){
-    sqlite3ErrorMsg(pParse,"LIMIT clause should come after %s not before",
-      selectOpName(p->op));
+  if( pPrior->pOrderBy || pPrior->pLimit ){
+    sqlite3ErrorMsg(pParse,"%s clause should come after %s not before",
+      pPrior->pOrderBy!=0 ? "ORDER BY" : "LIMIT", selectOpName(p->op));
     rc = 1;
     goto multi_select_end;
   }

From 460f1fa55cd4fdca39977f5f38e173f2ac44bace Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 18 Sep 2017 16:28:56 +0000
Subject: [PATCH 100/270] Add extension "mmapwarm.c". Provides function
 sqlite3_mmap_warm(), used to "warm up" the memory mapping used by SQLite in
 mmap mode to access db file content.

FossilOrigin-Name: d4a30b91f9aad93510baead8c04ee51b82c98763be5a224ed4873298214c963a
---
 Makefile.in         |   1 +
 Makefile.msc        |   1 +
 ext/misc/mmapwarm.c | 108 ++++++++++++++++++++++++++++++++++++++++++++
 main.mk             |   1 +
 manifest            |  24 ++++++----
 manifest.uuid       |   2 +-
 src/test1.c         |  34 +++++++++++++-
 7 files changed, 158 insertions(+), 13 deletions(-)
 create mode 100644 ext/misc/mmapwarm.c

diff --git a/Makefile.in b/Makefile.in
index 50cd5e8e23..58c2c2919b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -428,6 +428,7 @@ TESTSRC += \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c \
   $(TOP)/ext/misc/ieee754.c \
+  $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
diff --git a/Makefile.msc b/Makefile.msc
index 1289fe6d5b..799024652d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1411,6 +1411,7 @@ TESTEXT = \
   $(TOP)\ext\fts5\fts5_test_mi.c \
   $(TOP)\ext\fts5\fts5_test_tok.c \
   $(TOP)\ext\misc\ieee754.c \
+  $(TOP)\ext\misc\mmapwarm.c
   $(TOP)\ext\misc\nextchar.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
diff --git a/ext/misc/mmapwarm.c b/ext/misc/mmapwarm.c
new file mode 100644
index 0000000000..4e23638a99
--- /dev/null
+++ b/ext/misc/mmapwarm.c
@@ -0,0 +1,108 @@
+/*
+** 2017-09-18
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+*/
+
+#include "sqlite3.h"
+
+
+/*
+** This function is used to touch each page of a mapping of a memory
+** mapped SQLite database. Assuming that the system has sufficient free
+** memory and supports sufficiently large mappings, this causes the OS 
+** to cache the entire database in main memory, making subsequent 
+** database accesses faster.
+**
+** If the second parameter to this function is not NULL, it is the name of
+** the specific database to operate on (i.e. "main" or the name of an
+** attached database).
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+** It is not considered an error if the file is not memory-mapped, or if
+** the mapping does not span the entire file. If an error does occur, a
+** transaction may be left open on the database file.
+**
+** It is illegal to call this function when the database handle has an 
+** open transaction. SQLITE_MISUSE is returned in this case.
+*/
+int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){
+  int rc = SQLITE_OK;
+  char *zSql = 0;
+  int pgsz = 0;
+  int nTotal = 0;
+
+  if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE;
+
+  /* Open a read-only transaction on the file in question */
+  zSql = sqlite3_mprintf("BEGIN; SELECT * FROM %s%q%ssqlite_master", 
+      (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
+  );
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+
+  /* Find the SQLite page size of the file */
+  if( rc==SQLITE_OK ){
+    zSql = sqlite3_mprintf("PRAGMA %s%q%spage_size", 
+        (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "")
+    );
+    if( zSql==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      sqlite3_stmt *pPgsz = 0;
+      rc = sqlite3_prepare_v2(db, zSql, -1, &pPgsz, 0);
+      sqlite3_free(zSql);
+      if( rc==SQLITE_OK ){
+        if( sqlite3_step(pPgsz)==SQLITE_ROW ){
+          pgsz = sqlite3_column_int(pPgsz, 0);
+        }
+        rc = sqlite3_finalize(pPgsz);
+      }
+      if( rc==SQLITE_OK && pgsz==0 ){
+        rc = SQLITE_ERROR;
+      }
+    }
+  }
+
+  /* Touch each mmap'd page of the file */
+  if( rc==SQLITE_OK ){
+    int rc2;
+    sqlite3_file *pFd = 0;
+    rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFd);
+    if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){
+      sqlite3_int64 iPg = 1;
+      sqlite3_io_methods const *p = pFd->pMethods;
+      while( 1 ){
+        unsigned char *pMap;
+        rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap);
+        if( rc!=SQLITE_OK || pMap==0 ) break;
+
+        nTotal += pMap[0];
+        nTotal += pMap[pgsz-1];
+
+        rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap);
+        if( rc!=SQLITE_OK ) break;
+        iPg++;
+      }
+      sqlite3_log(SQLITE_OK, 
+          "sqlite3_mmap_warm_cache: Warmed up %d pages of %s", iPg==1?0:iPg,
+          sqlite3_db_filename(db, zDb)
+      );
+    }
+
+    rc2 = sqlite3_exec(db, "END", 0, 0, 0);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  return rc;
+}
+
diff --git a/main.mk b/main.mk
index 054f6518a5..7da6db15f1 100644
--- a/main.mk
+++ b/main.mk
@@ -334,6 +334,7 @@ TESTSRC += \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
   $(TOP)/ext/misc/ieee754.c \
+  $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
diff --git a/manifest b/manifest
index 3c256819da..d673fb237f 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sthe\sCSV\svirtual\stable\sextension\sso\sthat\sit\sworks\swhen\sthe\sdefault\scharacter\nis\sunsigned.
-D 2017-09-18T00:18:31.200
-F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
+C Add\sextension\s"mmapwarm.c".\sProvides\sfunction\ssqlite3_mmap_warm(),\sused\sto\n"warm\sup"\sthe\smemory\smapping\sused\sby\sSQLite\sin\smmap\smode\sto\saccess\sdb\sfile\ncontent.
+D 2017-09-18T16:28:56.386
+F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b
+F Makefile.msc 2a1cf3959b03b3cffedcbc3cf6bde8c992ed865667f2a7aefb14cb2105ec97c7
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -270,6 +270,7 @@ F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33
+F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
@@ -381,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk da75a0527a56da0b7f568a976b3cb69756613080f16e4d208b6c6a0495bfb132
+F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -468,7 +469,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828
-F src/test1.c 8513b17ca4a7a9ba28748535d178b6e472ec7394ae0eea53907f2d3bcdbab2df
+F src/test1.c a947b2554fa77d0ef2dd21d1ef08e37e5d91b17af83de923a4e3c7f10957a2eb
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1654,7 +1655,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 29227d00a9999f0f28a0b55ef70183799a667c3b9d81d2e5ac0ab1840bef98b1
-R ccebe3b4b57d7dc8c5482ee2b9f63107
-U drh
-Z a535a0ac2d4006e12331e0812c689362
+P 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
+R 2de3e498b6d544466fabd508443a98d0
+T *branch * mmap-warm
+T *sym-mmap-warm *
+T -sym-trunk *
+U dan
+Z 5972d569bdc42461a8703da92c49c171
diff --git a/manifest.uuid b/manifest.uuid
index 811612d113..17ca822d09 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
\ No newline at end of file
+d4a30b91f9aad93510baead8c04ee51b82c98763be5a224ed4873298214c963a
\ No newline at end of file
diff --git a/src/test1.c b/src/test1.c
index 7a6ff2163f..d70ce77a7a 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -7415,6 +7415,35 @@ static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
   }
 }
 
+/*
+** Usage: sqlite3_mmap_warm DB DBNAME
+*/
+static int SQLITE_TCLAPI test_mmap_warm(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**);
+  extern int sqlite3_mmap_warm(sqlite3 *db, const char *);
+
+  if( objc!=2 && objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB ?DBNAME?");
+    return TCL_ERROR;
+  }else{
+    int rc;
+    sqlite3 *db;
+    const char *zDb = 0;
+    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+    if( objc==3 ){
+      zDb = Tcl_GetString(objv[2]);
+    }
+    rc = sqlite3_mmap_warm(db, zDb);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_OK;
+  }
+}
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -7684,8 +7713,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
      { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
 #endif
-     { "sqlite3_delete_database", test_delete_database, 0 },
-     { "atomic_batch_write",      test_atomic_batch_write,     0   },
+     { "sqlite3_delete_database", test_delete_database,    0 },
+     { "atomic_batch_write",      test_atomic_batch_write, 0 },
+     { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
   };
   static int bitmask_size = sizeof(Bitmask)*8;
   static int longdouble_size = sizeof(LONGDOUBLE_TYPE);

From 262d55121ff91fd4595258d3be4cb1e0ccf20173 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 18 Sep 2017 18:08:54 +0000
Subject: [PATCH 101/270] Fix a typo in the MSVC makefile.

FossilOrigin-Name: 3235835babb49b4dd1acaabd1aa6cfb0b7fe19a914db1cb511e8cc872d3c0c39
---
 Makefile.msc  |  2 +-
 manifest      | 17 +++++++----------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 799024652d..10585e4a58 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1411,7 +1411,7 @@ TESTEXT = \
   $(TOP)\ext\fts5\fts5_test_mi.c \
   $(TOP)\ext\fts5\fts5_test_tok.c \
   $(TOP)\ext\misc\ieee754.c \
-  $(TOP)\ext\misc\mmapwarm.c
+  $(TOP)\ext\misc\mmapwarm.c \
   $(TOP)\ext\misc\nextchar.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
diff --git a/manifest b/manifest
index d673fb237f..7da7e371b3 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Add\sextension\s"mmapwarm.c".\sProvides\sfunction\ssqlite3_mmap_warm(),\sused\sto\n"warm\sup"\sthe\smemory\smapping\sused\sby\sSQLite\sin\smmap\smode\sto\saccess\sdb\sfile\ncontent.
-D 2017-09-18T16:28:56.386
+C Fix\sa\stypo\sin\sthe\sMSVC\smakefile.
+D 2017-09-18T18:08:54.087
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2a1cf3959b03b3cffedcbc3cf6bde8c992ed865667f2a7aefb14cb2105ec97c7
+F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1655,10 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 42f07775556758754e92e29a759d200d0d81d16eee83ab982b840db11292f834
-R 2de3e498b6d544466fabd508443a98d0
-T *branch * mmap-warm
-T *sym-mmap-warm *
-T -sym-trunk *
-U dan
-Z 5972d569bdc42461a8703da92c49c171
+P d4a30b91f9aad93510baead8c04ee51b82c98763be5a224ed4873298214c963a
+R 4ba39d2b6ec2c2ddd6b331aa8e33693e
+U drh
+Z e0cd459a48e910a5c5781b8e3cab411d
diff --git a/manifest.uuid b/manifest.uuid
index 17ca822d09..f956c6b7ad 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4a30b91f9aad93510baead8c04ee51b82c98763be5a224ed4873298214c963a
\ No newline at end of file
+3235835babb49b4dd1acaabd1aa6cfb0b7fe19a914db1cb511e8cc872d3c0c39
\ No newline at end of file

From 9a243e69c2a067040aa2fe373cb25ec1278e9112 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Sep 2017 09:09:34 +0000
Subject: [PATCH 102/270] Updates to the "lemon.html" document received from
 Andy Goth.

FossilOrigin-Name: 5b2002f3df1902aaa571a0efd01ab8bae7f4d37ac4819cc51595277f4de93433
---
 doc/lemon.html | 379 ++++++++++++++++++++++++++-----------------------
 manifest       |  13 +-
 manifest.uuid  |   2 +-
 3 files changed, 210 insertions(+), 184 deletions(-)

diff --git a/doc/lemon.html b/doc/lemon.html
index f05c481d79..3ed85176f7 100644
--- a/doc/lemon.html
+++ b/doc/lemon.html
@@ -2,12 +2,12 @@
 <head>
 <title>The Lemon Parser Generator</title>
 </head>
-<body bgcolor=white>
-<h1 align=center>The Lemon Parser Generator</h1>
+<body bgcolor='white'>
+<h1 align='center'>The Lemon Parser Generator</h1>
 
 <p>Lemon is an LALR(1) parser generator for C.
 It does the same job as "bison" and "yacc".
-But lemon is not a bison or yacc clone.  Lemon
+But Lemon is not a bison or yacc clone.  Lemon
 uses a different grammar syntax which is designed to
 reduce the number of coding errors.  Lemon also uses a
 parsing engine that is faster than yacc and
@@ -16,7 +16,7 @@ bison and which is both reentrant and threadsafe.
 has also been updated so that it too can generate a
 reentrant and threadsafe parser.)
 Lemon also implements features that can be used
-to eliminate resource leaks, making is suitable for use
+to eliminate resource leaks, making it suitable for use
 in long-running programs such as graphical user interfaces
 or embedded controllers.</p>
 
@@ -58,8 +58,8 @@ Lemon comes with a default parser template which works fine for most
 applications.  But the user is free to substitute a different parser
 template if desired.</p>
 
-<p>Depending on command-line options, Lemon will generate between
-one and three files of outputs.
+<p>Depending on command-line options, Lemon will generate up to
+three output files.
 <ul>
 <li>C code to implement the parser.
 <li>A header file defining an integer ID for each terminal symbol.
@@ -90,17 +90,20 @@ the states used by the parser automaton.</p>
 You can obtain a list of the available command-line options together
 with a brief explanation of what each does by typing
 <pre>
-   lemon -?
+   lemon "-?"
 </pre>
 As of this writing, the following command-line options are supported:
 <ul>
 <li><b>-b</b>
 Show only the basis for each parser state in the report file.
 <li><b>-c</b>
-Do not compress the generated action tables.
+Do not compress the generated action tables.  The parser will be a
+little larger and slower, but it will detect syntax errors sooner.
 <li><b>-D<i>name</i></b>
-Define C preprocessor macro <i>name</i>.  This macro is useable by
-"%ifdef" lines in the grammar file.
+Define C preprocessor macro <i>name</i>.  This macro is usable by
+"<tt><a href='#pifdef'>%ifdef</a></tt>" and
+"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
+in the grammar file.
 <li><b>-g</b>
 Do not generate a parser.  Instead write the input grammar to standard
 output with all comments, actions, and other extraneous text removed.
@@ -108,9 +111,9 @@ output with all comments, actions, and other extraneous text removed.
 Omit "#line" directives in the generated parser C code.
 <li><b>-m</b>
 Cause the output C source code to be compatible with the "makeheaders"
-program. 
+program.
 <li><b>-p</b>
-Display all conflicts that are resolved by 
+Display all conflicts that are resolved by
 <a href='#precrules'>precedence rules</a>.
 <li><b>-q</b>
 Suppress generation of the report file.
@@ -165,7 +168,7 @@ once for each token:
 </pre>
 The first argument to the Parse() routine is the pointer returned by
 ParseAlloc().
-The second argument is a small positive integer that tells the parse the
+The second argument is a small positive integer that tells the parser the
 type of the next token in the data stream.
 There is one token type for each terminal symbol in the grammar.
 The gram.h file generated by Lemon contains #define statements that
@@ -173,7 +176,7 @@ map symbolic terminal symbol names into appropriate integer values.
 A value of 0 for the second argument is a special flag to the
 parser to indicate that the end of input has been reached.
 The third argument is the value of the given token.  By default,
-the type of the third argument is integer, but the grammar will
+the type of the third argument is "void*", but the grammar will
 usually redefine this type to be some kind of structure.
 Typically the second argument will be a broad category of tokens
 such as "identifier" or "number" and the third argument will
@@ -181,7 +184,7 @@ be the name of the identifier or the value of the number.</p>
 
 <p>The Parse() function may have either three or four arguments,
 depending on the grammar.  If the grammar specification file requests
-it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
+it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
 the Parse() function will have a fourth parameter that can be
 of any type chosen by the programmer.  The parser doesn't do anything
 with this argument except to pass it through to action routines.
@@ -191,20 +194,20 @@ to the action routines without having to use global variables.</p>
 <p>A typical use of a Lemon parser might look something like the
 following:
 <pre>
-   01 ParseTree *ParseFile(const char *zFilename){
-   02    Tokenizer *pTokenizer;
-   03    void *pParser;
-   04    Token sToken;
-   05    int hTokenId;
-   06    ParserState sState;
-   07
-   08    pTokenizer = TokenizerCreate(zFilename);
-   09    pParser = ParseAlloc( malloc );
-   10    InitParserState(&sState);
-   11    while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
-   12       Parse(pParser, hTokenId, sToken, &sState);
+    1 ParseTree *ParseFile(const char *zFilename){
+    2    Tokenizer *pTokenizer;
+    3    void *pParser;
+    4    Token sToken;
+    5    int hTokenId;
+    6    ParserState sState;
+    7
+    8    pTokenizer = TokenizerCreate(zFilename);
+    9    pParser = ParseAlloc( malloc );
+   10    InitParserState(&amp;sState);
+   11    while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
+   12       Parse(pParser, hTokenId, sToken, &amp;sState);
    13    }
-   14    Parse(pParser, 0, sToken, &sState);
+   14    Parse(pParser, 0, sToken, &amp;sState);
    15    ParseFree(pParser, free );
    16    TokenizerFree(pTokenizer);
    17    return sState.treeRoot;
@@ -217,10 +220,10 @@ simple.)
 We assume the existence of some kind of tokenizer which is created
 using TokenizerCreate() on line 8 and deleted by TokenizerFree()
 on line 16.  The GetNextToken() function on line 11 retrieves the
-next token from the input file and puts its type in the 
+next token from the input file and puts its type in the
 integer variable hTokenId.  The sToken variable is assumed to be
 some kind of structure that contains details about each token,
-such as its complete text, what line it occurs on, etc. </p>
+such as its complete text, what line it occurs on, etc.</p>
 
 <p>This example also assumes the existence of structure of type
 ParserState that holds state information about a particular parse.
@@ -237,7 +240,7 @@ tree.</p>
 <pre>
    ParseFile(){
       pParser = ParseAlloc( malloc );
-      while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
+      while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
          Parse(pParser, hTokenId, sToken);
       }
       Parse(pParser, 0, sToken);
@@ -297,25 +300,25 @@ specifies additional information Lemon requires to do its job.
 Most of the work in using Lemon is in writing an appropriate
 grammar file.</p>
 
-<p>The grammar file for lemon is, for the most part, free format.
+<p>The grammar file for Lemon is, for the most part, free format.
 It does not have sections or divisions like yacc or bison.  Any
 declaration can occur at any point in the file.
 Lemon ignores whitespace (except where it is needed to separate
-tokens) and it honors the same commenting conventions as C and C++.</p>
+tokens), and it honors the same commenting conventions as C and C++.</p>
 
 <h3>Terminals and Nonterminals</h3>
 
 <p>A terminal symbol (token) is any string of alphanumeric
 and/or underscore characters
-that begins with an upper case letter.
+that begins with an uppercase letter.
 A terminal can contain lowercase letters after the first character,
-but the usual convention is to make terminals all upper case.
+but the usual convention is to make terminals all uppercase.
 A nonterminal, on the other hand, is any string of alphanumeric
-and underscore characters than begins with a lower case letter.
-Again, the usual convention is to make nonterminals use all lower
-case letters.</p>
+and underscore characters than begins with a lowercase letter.
+Again, the usual convention is to make nonterminals use all lowercase
+letters.</p>
 
-<p>In Lemon, terminal and nonterminal symbols do not need to 
+<p>In Lemon, terminal and nonterminal symbols do not need to
 be declared or identified in a separate section of the grammar file.
 Lemon is able to generate a list of all terminals and nonterminals
 by examining the grammar rules, and it can always distinguish a
@@ -339,7 +342,8 @@ The list of terminals and nonterminals on the right-hand side of the
 rule can be empty.
 Rules can occur in any order, except that the left-hand side of the
 first rule is assumed to be the start symbol for the grammar (unless
-specified otherwise using the <tt>%start</tt> directive described below.)
+specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
+directive described below.)
 A typical sequence of grammar rules might look something like this:
 <pre>
   expr ::= expr PLUS expr.
@@ -382,7 +386,7 @@ names to each symbol in a grammar rule and then using those symbolic
 names in the action.
 In yacc or bison, one would write this:
 <pre>
-  expr -> expr PLUS expr  { $$ = $1 + $3; };
+  expr -&gt; expr PLUS expr  { $$ = $1 + $3; };
 </pre>
 But in Lemon, the same rule becomes the following:
 <pre>
@@ -422,14 +426,14 @@ of the shift, and a reduce-reduce conflict is resolved by reducing
 whichever rule comes first in the grammar file.</p>
 
 <p>Just like in
-yacc and bison, Lemon allows a measure of control 
-over the resolution of paring conflicts using precedence rules.
+yacc and bison, Lemon allows a measure of control
+over the resolution of parsing conflicts using precedence rules.
 A precedence value can be assigned to any terminal symbol
-using the 
-<a href='#pleft'>%left</a>,
-<a href='#pright'>%right</a> or
-<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
-mentioned in earlier directives have a lower precedence that
+using the
+<tt><a href='#pleft'>%left</a></tt>,
+<tt><a href='#pright'>%right</a></tt> or
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives.  Terminal symbols
+mentioned in earlier directives have a lower precedence than
 terminal symbols mentioned in later directives.  For example:</p>
 
 <p><pre>
@@ -505,29 +509,29 @@ as follows:
 <li> If the precedence of the token to be shifted is greater than
      the precedence of the rule to reduce, then resolve in favor
      of the shift.  No parsing conflict is reported.
-<li> If the precedence of the token it be shifted is less than the
+<li> If the precedence of the token to be shifted is less than the
      precedence of the rule to reduce, then resolve in favor of the
      reduce action.  No parsing conflict is reported.
 <li> If the precedences are the same and the shift token is
      right-associative, then resolve in favor of the shift.
      No parsing conflict is reported.
-<li> If the precedences are the same the shift token is
+<li> If the precedences are the same and the shift token is
      left-associative, then resolve in favor of the reduce.
      No parsing conflict is reported.
-<li> Otherwise, resolve the conflict by doing the shift and
-     report the parsing conflict.
+<li> Otherwise, resolve the conflict by doing the shift, and
+     report a parsing conflict.
 </ul>
 Reduce-reduce conflicts are resolved this way:
 <ul>
-<li> If either reduce rule 
+<li> If either reduce rule
      lacks precedence information, then resolve in favor of the
-     rule that appears first in the grammar and report a parsing
+     rule that appears first in the grammar, and report a parsing
      conflict.
-<li> If both rules have precedence and the precedence is different
+<li> If both rules have precedence and the precedence is different,
      then resolve the dispute in favor of the rule with the highest
-     precedence and do not report a conflict.
+     precedence, and do not report a conflict.
 <li> Otherwise, resolve the conflict by reducing by the rule that
-     appears first in the grammar and report a parsing conflict.
+     appears first in the grammar, and report a parsing conflict.
 </ul>
 
 <h3>Special Directives</h3>
@@ -536,40 +540,40 @@ Reduce-reduce conflicts are resolved this way:
 directives.  We've described all the grammar rules, so now we'll
 talk about the special directives.</p>
 
-<p>Directives in lemon can occur in any order.  You can put them before
-the grammar rules, or after the grammar rules, or in the mist of the
+<p>Directives in Lemon can occur in any order.  You can put them before
+the grammar rules, or after the grammar rules, or in the midst of the
 grammar rules.  It doesn't matter.  The relative order of
 directives used to assign precedence to terminals is important, but
 other than that, the order of directives in Lemon is arbitrary.</p>
 
 <p>Lemon supports the following special directives:
 <ul>
-<li><tt>%code</tt>
-<li><tt>%default_destructor</tt>
-<li><tt>%default_type</tt>
-<li><tt>%destructor</tt>
-<li><tt>%endif</tt>
-<li><tt>%extra_argument</tt>
-<li><tt>%fallback</tt>
-<li><tt>%ifdef</tt>
-<li><tt>%ifndef</tt>
-<li><tt>%include</tt>
-<li><tt>%left</tt>
-<li><tt>%name</tt>
-<li><tt>%nonassoc</tt>
-<li><tt>%parse_accept</tt>
-<li><tt>%parse_failure </tt>
-<li><tt>%right</tt>
-<li><tt>%stack_overflow</tt>
-<li><tt>%stack_size</tt>
-<li><tt>%start_symbol</tt>
-<li><tt>%syntax_error</tt>
-<li><tt>%token_class</tt>
-<li><tt>%token_destructor</tt>
-<li><tt>%token_prefix</tt>
-<li><tt>%token_type</tt>
-<li><tt>%type</tt>
-<li><tt>%wildcard</tt>
+<li><tt><a href='#pcode'>%code</a></tt>
+<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
+<li><tt><a href='#default_type'>%default_type</a></tt>
+<li><tt><a href='#destructor'>%destructor</a></tt>
+<li><tt><a href='#pifdef'>%endif</a></tt>
+<li><tt><a href='#extraarg'>%extra_argument</a></tt>
+<li><tt><a href='#pfallback'>%fallback</a></tt>
+<li><tt><a href='#pifdef'>%ifdef</a></tt>
+<li><tt><a href='#pifdef'>%ifndef</a></tt>
+<li><tt><a href='#pinclude'>%include</a></tt>
+<li><tt><a href='#pleft'>%left</a></tt>
+<li><tt><a href='#pname'>%name</a></tt>
+<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
+<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
+<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
+<li><tt><a href='#pright'>%right</a></tt>
+<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
+<li><tt><a href='#stack_size'>%stack_size</a></tt>
+<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
+<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
+<li><tt><a href='#token_class'>%token_class</a></tt>
+<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
+<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
+<li><tt><a href='#token_type'>%token_type</a></tt>
+<li><tt><a href='#ptype'>%type</a></tt>
+<li><tt><a href='#pwildcard'>%wildcard</a></tt>
 </ul>
 Each of these directives will be described separately in the
 following sections:</p>
@@ -577,43 +581,42 @@ following sections:</p>
 <a name='pcode'></a>
 <h4>The <tt>%code</tt> directive</h4>
 
-<p>The %code directive is used to specify addition C code that
+<p>The <tt>%code</tt> directive is used to specify additional C code that
 is added to the end of the main output file.  This is similar to
-the <a href='#pinclude'>%include</a> directive except that %include
-is inserted at the beginning of the main output file.</p>
+the <tt><a href='#pinclude'>%include</a></tt> directive except that
+<tt>%include</tt> is inserted at the beginning of the main output file.</p>
 
-<p>%code is typically used to include some action routines or perhaps
-a tokenizer or even the "main()" function 
+<p><tt>%code</tt> is typically used to include some action routines or perhaps
+a tokenizer or even the "main()" function
 as part of the output file.</p>
 
 <a name='default_destructor'></a>
 <h4>The <tt>%default_destructor</tt> directive</h4>
 
-<p>The %default_destructor directive specifies a destructor to 
+<p>The <tt>%default_destructor</tt> directive specifies a destructor to
 use for non-terminals that do not have their own destructor
-specified by a separate %destructor directive.  See the documentation
-on the <a name='#destructor'>%destructor</a> directive below for
+specified by a separate <tt>%destructor</tt> directive.  See the documentation
+on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
 additional information.</p>
 
-<p>In some grammers, many different non-terminal symbols have the
-same datatype and hence the same destructor.  This directive is
-a convenience way to specify the same destructor for all those
+<p>In some grammars, many different non-terminal symbols have the
+same data type and hence the same destructor.  This directive is
+a convenient way to specify the same destructor for all those
 non-terminals using a single statement.</p>
 
 <a name='default_type'></a>
 <h4>The <tt>%default_type</tt> directive</h4>
 
-<p>The %default_type directive specifies the datatype of non-terminal
-symbols that do no have their own datatype defined using a separate
-<a href='#ptype'>%type</a> directive.  
-</p>
+<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
+symbols that do not have their own data type defined using a separate
+<tt><a href='#ptype'>%type</a></tt> directive.</p>
 
 <a name='destructor'></a>
 <h4>The <tt>%destructor</tt> directive</h4>
 
-<p>The %destructor directive is used to specify a destructor for
+<p>The <tt>%destructor</tt> directive is used to specify a destructor for
 a non-terminal symbol.
-(See also the <a href='#token_destructor'>%token_destructor</a>
+(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
 directive which is used to specify a destructor for terminal symbols.)</p>
 
 <p>A non-terminal's destructor is called to dispose of the
@@ -635,7 +638,7 @@ or other resources held by that non-terminal.</p>
    %destructor nt { free($$); }
    nt(A) ::= ID NUM.   { A = malloc( 100 ); }
 </pre>
-This example is a bit contrived but it serves to illustrate how
+This example is a bit contrived, but it serves to illustrate how
 destructors work.  The example shows a non-terminal named
 "nt" that holds values of type "void*".  When the rule for
 an "nt" reduces, it sets the value of the non-terminal to
@@ -651,17 +654,17 @@ stack, unless the non-terminal is used in a C-code action.  If
 the non-terminal is used by C-code, then it is assumed that the
 C-code will take care of destroying it.
 More commonly, the value is used to build some
-larger structure and we don't want to destroy it, which is why
+larger structure, and we don't want to destroy it, which is why
 the destructor is not called in this circumstance.</p>
 
 <p>Destructors help avoid memory leaks by automatically freeing
 allocated objects when they go out of scope.
 To do the same using yacc or bison is much more difficult.</p>
 
-<a name="extraarg"></a>
+<a name='extraarg'></a>
 <h4>The <tt>%extra_argument</tt> directive</h4>
 
-The %extra_argument directive instructs Lemon to add a 4th parameter
+The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
 to the parameter list of the Parse() function it generates.  Lemon
 doesn't do anything itself with this extra argument, but it does
 make the argument available to C-code action routines, destructors,
@@ -679,61 +682,64 @@ in the most recent call to Parse().</p>
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>
 
-<p>The %fallback directive specifies an alternative meaning for one
+<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
 or more tokens.  The alternative meaning is tried if the original token
-would have generated a syntax error.
+would have generated a syntax error.</p>
 
-<p>The %fallback directive was added to support robust parsing of SQL
-syntax in <a href="https://www.sqlite.org/">SQLite</a>.
+<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
+syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
 The SQL language contains a large assortment of keywords, each of which
 appears as a different token to the language parser.  SQL contains so
-many keywords, that it can be difficult for programmers to keep up with
+many keywords that it can be difficult for programmers to keep up with
 them all.  Programmers will, therefore, sometimes mistakenly use an
-obscure language keyword for an identifier.  The %fallback directive
+obscure language keyword for an identifier.  The <tt>%fallback</tt> directive
 provides a mechanism to tell the parser:  "If you are unable to parse
-this keyword, try treating it as an identifier instead."
+this keyword, try treating it as an identifier instead."</p>
 
-<p>The syntax of %fallback is as follows:
+<p>The syntax of <tt>%fallback</tt> is as follows:
 
 <blockquote>
-<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
-</blockquote>
+<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
+</blockquote></p>
 
-<p>In words, the %fallback directive is followed by a list of token names
-terminated by a period.  The first token name is the fallback token - the
+<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
+names terminated by a period.
+The first token name is the fallback token &mdash; the
 token to which all the other tokens fall back to.  The second and subsequent
 arguments are tokens which fall back to the token identified by the first
-argument.
+argument.</p>
 
 <a name='pifdef'></a>
-<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
+<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>
 
-<p>The %ifdef, %ifndef, and %endif directives are similar to
-#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
+<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
+are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
+just not as general.
 Each of these directives must begin at the left margin.  No whitespace
-is allowed between the "%" and the directive name.
+is allowed between the "%" and the directive name.</p>
 
-<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
+<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
+"<tt>%endif</tt>" is
 ignored unless the "-DMACRO" command-line option is used.  Grammar text
-betwen "%ifndef MACRO" and the next nested "%endif" is included except when
-the "-DMACRO" command-line option is used.
+betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
+included except when the "-DMACRO" command-line option is used.</p>
 
-<p>Note that the argument to %ifdef and %ifndef must be a single 
-preprocessor symbol name, not a general expression.  There is no "%else"
-directive.
+<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
+be a single preprocessor symbol name, not a general expression.
+There is no "<tt>%else</tt>" directive.</p>
 
 
 <a name='pinclude'></a>
 <h4>The <tt>%include</tt> directive</h4>
 
-<p>The %include directive specifies C code that is included at the
-top of the generated parser.  You can include any text you want --
+<p>The <tt>%include</tt> directive specifies C code that is included at the
+top of the generated parser.  You can include any text you want &mdash;
 the Lemon parser generator copies it blindly.  If you have multiple
-%include directives in your grammar file, their values are concatenated
-so that all %include code ultimately appears near the top of the
-generated parser, in the same order as it appeared in the grammer.</p>
+<tt>%include</tt> directives in your grammar file, their values are concatenated
+so that all <tt>%include</tt> code ultimately appears near the top of the
+generated parser, in the same order as it appeared in the grammar.</p>
 
-<p>The %include directive is very handy for getting some extra #include
+<p>The <tt>%include</tt> directive is very handy for getting some extra #include
 preprocessor statements at the beginning of the generated parser.
 For example:</p>
 
@@ -742,17 +748,19 @@ For example:</p>
 </pre></p>
 
 <p>This might be needed, for example, if some of the C actions in the
-grammar call functions that are prototyed in unistd.h.</p>
+grammar call functions that are prototyped in unistd.h.</p>
 
 <a name='pleft'></a>
 <h4>The <tt>%left</tt> directive</h4>
 
-The %left directive is used (along with the <a href='#pright'>%right</a> and
-<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
-terminal symbols.  Every terminal symbol whose name appears after
-a %left directive but before the next period (".") is
+The <tt>%left</tt> directive is used (along with the
+<tt><a href='#pright'>%right</a></tt> and
+<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
+precedences of terminal symbols.
+Every terminal symbol whose name appears after
+a <tt>%left</tt> directive but before the next period (".") is
 given the same left-associative precedence value.  Subsequent
-%left directives have higher precedence.  For example:</p>
+<tt>%left</tt> directives have higher precedence.  For example:</p>
 
 <p><pre>
    %left AND.
@@ -763,20 +771,21 @@ given the same left-associative precedence value.  Subsequent
    %right EXP NOT.
 </pre></p>
 
-<p>Note the period that terminates each %left, %right or %nonassoc
+<p>Note the period that terminates each <tt>%left</tt>,
+<tt>%right</tt> or <tt>%nonassoc</tt>
 directive.</p>
 
 <p>LALR(1) grammars can get into a situation where they require
 a large amount of stack space if you make heavy use or right-associative
-operators.  For this reason, it is recommended that you use %left
-rather than %right whenever possible.</p>
+operators.  For this reason, it is recommended that you use <tt>%left</tt>
+rather than <tt>%right</tt> whenever possible.</p>
 
 <a name='pname'></a>
 <h4>The <tt>%name</tt> directive</h4>
 
 <p>By default, the functions generated by Lemon all begin with the
 five-character string "Parse".  You can change this string to something
-different using the %name directive.  For instance:</p>
+different using the <tt>%name</tt> directive.  For instance:</p>
 
 <p><pre>
    %name Abcde
@@ -790,22 +799,22 @@ functions named
 <li> AbcdeTrace(), and
 <li> Abcde().
 </ul>
-The %name directive allows you to generator two or more different
-parsers and link them all into the same executable.
-</p>
+The <tt>%name</tt> directive allows you to generate two or more different
+parsers and link them all into the same executable.</p>
 
 <a name='pnonassoc'></a>
 <h4>The <tt>%nonassoc</tt> directive</h4>
 
 <p>This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
-or on the <a href='#pleft'>%left</a> directive for additional information.</p>
+or on the <tt><a href='#pleft'>%left</a></tt> directive
+for additional information.</p>
 
 <a name='parse_accept'></a>
 <h4>The <tt>%parse_accept</tt> directive</h4>
 
-<p>The %parse_accept directive specifies a block of C code that is
+<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
 executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.</p>
@@ -821,7 +830,7 @@ without error.</p>
 <a name='parse_failure'></a>
 <h4>The <tt>%parse_failure</tt> directive</h4>
 
-<p>The %parse_failure directive specifies a block of C code that
+<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
 is executed whenever the parser fails complete.  This code is not
 executed until the parser has tried and failed to resolve an input
 error using is usual error recovery strategy.  The routine is
@@ -837,14 +846,14 @@ only invoked when parsing is unable to continue.</p>
 <h4>The <tt>%right</tt> directive</h4>
 
 <p>This directive is used to assign right-associative precedence to
-one or more terminal symbols.  See the section on 
+one or more terminal symbols.  See the section on
 <a href='#precrules'>precedence rules</a>
 or on the <a href='#pleft'>%left</a> directive for additional information.</p>
 
 <a name='stack_overflow'></a>
 <h4>The <tt>%stack_overflow</tt> directive</h4>
 
-<p>The %stack_overflow directive specifies a block of C code that
+<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
 is executed if the parser's internal stack ever overflows.  Typically
 this just prints an error message.  After a stack overflow, the parser
 will be unable to continue and must be reset.</p>
@@ -857,7 +866,7 @@ will be unable to continue and must be reset.</p>
 
 <p>You can help prevent parser stack overflows by avoiding the use
 of right recursion and right-precedence operators in your grammar.
-Use left recursion and and left-precedence operators instead, to
+Use left recursion and and left-precedence operators instead to
 encourage rules to reduce sooner and keep the stack size down.
 For example, do rules like this:
 <pre>
@@ -868,7 +877,7 @@ Not like this:
 <pre>
    list ::= element list.      // right-recursion.  Bad!
    list ::= .
-</pre>
+</pre></p>
 
 <a name='stack_size'></a>
 <h4>The <tt>%stack_size</tt> directive</h4>
@@ -876,7 +885,7 @@ Not like this:
 <p>If stack overflow is a problem and you can't resolve the trouble
 by using left-recursion, then you might want to increase the size
 of the parser's stack using this directive.  Put an positive integer
-after the %stack_size directive and Lemon will generate a parse
+after the <tt>%stack_size</tt> directive and Lemon will generate a parse
 with a stack of the requested size.  The default value is 100.</p>
 
 <p><pre>
@@ -886,25 +895,40 @@ with a stack of the requested size.  The default value is 100.</p>
 <a name='start_symbol'></a>
 <h4>The <tt>%start_symbol</tt> directive</h4>
 
-<p>By default, the start-symbol for the grammar that Lemon generates
+<p>By default, the start symbol for the grammar that Lemon generates
 is the first non-terminal that appears in the grammar file.  But you
-can choose a different start-symbol using the %start_symbol directive.</p>
+can choose a different start symbol using the
+<tt>%start_symbol</tt> directive.</p>
 
 <p><pre>
    %start_symbol  prog
 </pre></p>
 
+<a name='syntax_error'></a>
+<h4>The <tt>%syntax_error</tt> directive</h4>
+
+<p>See <a href='#error_processing'>Error Processing</a>.</p>
+
+<a name='token_class'></a>
+<h4>The <tt>%token_class</tt> directive</h4>
+
+<p>Undocumented.  Appears to be related to the MULTITERMINAL concept.
+<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
+
 <a name='token_destructor'></a>
 <h4>The <tt>%token_destructor</tt> directive</h4>
 
-<p>The %destructor directive assigns a destructor to a non-terminal
-symbol.  (See the description of the %destructor directive above.)
-This directive does the same thing for all terminal symbols.</p>
+<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
+symbol.  (See the description of the
+<tt><a href='%destructor'>%destructor</a></tt> directive above.)
+The <tt>%token_destructor</tt> directive does the same thing
+for all terminal symbols.</p>
 
 <p>Unlike non-terminal symbols which may each have a different data type
 for their values, terminals all use the same data type (defined by
-the %token_type directive) and so they use a common destructor.  Other
-than that, the token destructor works just like the non-terminal
+the <tt><a href='#token_type'>%token_type</a></tt> directive)
+and so they use a common destructor.
+Other than that, the token destructor works just like the non-terminal
 destructors.</p>
 
 <a name='token_prefix'></a>
@@ -913,8 +937,9 @@ destructors.</p>
 <p>Lemon generates #defines that assign small integer constants
 to each terminal symbol in the grammar.  If desired, Lemon will
 add a prefix specified by this directive
-to each of the #defines it generates.
-So if the default output of Lemon looked like this:
+to each of the #defines it generates.</p>
+
+<p>So if the default output of Lemon looked like this:
 <pre>
     #define AND              1
     #define MINUS            2
@@ -931,7 +956,7 @@ to cause Lemon to produce these symbols instead:
     #define TOKEN_MINUS      2
     #define TOKEN_OR         3
     #define TOKEN_PLUS       4
-</pre>
+</pre></p>
 
 <a name='token_type'></a><a name='ptype'></a>
 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
@@ -952,7 +977,7 @@ token structure.  Like this:</p>
 is "void*".</p>
 
 <p>Non-terminal symbols can each have their own data types.  Typically
-the data type  of a non-terminal is a pointer to the root of a parse-tree
+the data type of a non-terminal is a pointer to the root of a parse tree
 structure that contains all information about that non-terminal.
 For example:</p>
 
@@ -973,14 +998,15 @@ and able to pay that price, fine.  You just need to know.</p>
 <a name='pwildcard'></a>
 <h4>The <tt>%wildcard</tt> directive</h4>
 
-<p>The %wildcard directive is followed by a single token name and a
-period.  This directive specifies that the identified token should 
-match any input token.
+<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
+period.  This directive specifies that the identified token should
+match any input token.</p>
 
 <p>When the generated parser has the choice of matching an input against
 the wildcard token and some other token, the other token is always used.
-The wildcard token is only matched if there are no other alternatives.
+The wildcard token is only matched if there are no alternatives.</p>
 
+<a name='error_processing'></a>
 <h3>Error Processing</h3>
 
 <p>After extensive experimentation over several years, it has been
@@ -988,19 +1014,20 @@ discovered that the error recovery strategy used by yacc is about
 as good as it gets.  And so that is what Lemon uses.</p>
 
 <p>When a Lemon-generated parser encounters a syntax error, it
-first invokes the code specified by the %syntax_error directive, if
+first invokes the code specified by the <tt>%syntax_error</tt> directive, if
 any.  It then enters its error recovery strategy.  The error recovery
 strategy is to begin popping the parsers stack until it enters a
 state where it is permitted to shift a special non-terminal symbol
 named "error".  It then shifts this non-terminal and continues
-parsing.  But the %syntax_error routine will not be called again
+parsing.  The <tt>%syntax_error</tt> routine will not be called again
 until at least three new tokens have been successfully shifted.</p>
 
 <p>If the parser pops its stack until the stack is empty, and it still
-is unable to shift the error symbol, then the %parse_failed routine
+is unable to shift the error symbol, then the
+<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
 is invoked and the parser resets itself to its start state, ready
 to begin parsing a new file.  This is what will happen at the very
-first syntax error, of course, if there are no instances of the 
+first syntax error, of course, if there are no instances of the
 "error" non-terminal in your grammar.</p>
 
 </body>
diff --git a/manifest b/manifest
index b883f00be7..7d332d8350 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_mmap_warm()\sfunction\sas\san\sextension\sin\sthe\sext/misc/mmapwarm.c\ssource\sfile.
-D 2017-09-18T18:17:01.889
+C Updates\sto\sthe\s"lemon.html"\sdocument\sreceived\sfrom\sAndy\sGoth.
+D 2017-09-20T09:09:34.192
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -33,7 +33,7 @@ F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
 F configure e691ad9b505f1f47bc5d99be9e1d49b1be9037e9cb3821c9b14c63c3d413d055 x
 F configure.ac bb85c1c53e952c8c7078a2f147eba613e0128b8b6e7780d64758d8fb29bcc695
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
-F doc/lemon.html 1f8b8d4c9f5cfe40e679fee279cc9eb2da8e6eb74ad406028538d7864cc4b6cb
+F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
 F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
@@ -1655,8 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a944719314e0ac2f1954b65668815769eba3ab3e39a74666293b8dea52a184b2 3235835babb49b4dd1acaabd1aa6cfb0b7fe19a914db1cb511e8cc872d3c0c39
-R 62da41337307696798754c70fb4a4da8
-T +closed 3235835babb49b4dd1acaabd1aa6cfb0b7fe19a914db1cb511e8cc872d3c0c39
+P 1b2de41453ac33de82f9cd6cbb92eee4fe184fb282c27e5efa5243c8cb239630
+R e0e8cf7279386534b02109d8fa18ad99
 U drh
-Z 9c4b90490d8e7ae619a7569445f78dbf
+Z 86920861ac015f347841db4c53c64a7b
diff --git a/manifest.uuid b/manifest.uuid
index eea4571e30..0ce91438a9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1b2de41453ac33de82f9cd6cbb92eee4fe184fb282c27e5efa5243c8cb239630
\ No newline at end of file
+5b2002f3df1902aaa571a0efd01ab8bae7f4d37ac4819cc51595277f4de93433
\ No newline at end of file

From 84d4f1a328dd47fcaf5c6899babbdb64aff89184 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Sep 2017 10:47:10 +0000
Subject: [PATCH 103/270] Improved resolution of large integer values in
 "CAST(x AS NUMERIC)".

FossilOrigin-Name: 7f2bd4ff45fba29528c18cac6da983bd9b164303525d3965056f5b40f85dc83f
---
 manifest         | 22 +++++++++++-----------
 manifest.uuid    |  2 +-
 src/expr.c       |  4 ++--
 src/main.c       |  2 +-
 src/util.c       | 38 ++++++++++++++++++++------------------
 src/vdbe.c       |  2 +-
 src/vdbemem.c    | 15 +++++++++++----
 test/e_expr.test | 44 ++++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 91 insertions(+), 38 deletions(-)

diff --git a/manifest b/manifest
index 7d332d8350..2342cdcc0c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\s"lemon.html"\sdocument\sreceived\sfrom\sAndy\sGoth.
-D 2017-09-20T09:09:34.192
+C Improved\sresolution\sof\slarge\sinteger\svalues\sin\s"CAST(x\sAS\sNUMERIC)".
+D 2017-09-20T10:47:10.365
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -411,7 +411,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 38a225057f5b7d6a2bc403d7a5d3a67f59ee57b73fe7c546221e3c53202a3731
+F src/expr.c 82fedd57c8ce9e7dc16a003ad4cd863308787d5b5cbd0f83263b37805a56319c
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -423,7 +423,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 34a58563358fe40979186124d1a3614b9a00c833124d7ebfa8e5d604ed1d2521
+F src/main.c 1c2307a9f7e3183232a2d049881026fe43bf25659857cc11a9a44898accd3200
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -524,15 +524,15 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
 F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c ece10cb34b4a62cbb3aab8d711b67e93a2765e0b81d0f72458cb801a3ac60703
+F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 0a0ef7f0759ee4c3ed5ff0c6e9641216d4b31ebb0fea9a7b5a9c4a86ce875ff3
+F src/vdbe.c 2664b89eb243c24bd8dd66a7a13b04b5c83f41897dc80266ad13fed4bb09966c
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 2f9d672af5260f0145787e1dc2c6985414987cc0dc575133a0dc17dda767d868
+F src/vdbemem.c 08f89979c4b359483acb21b044d8e47db55f071ca4326e8bfdca5ac4cb5770ac
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -732,7 +732,7 @@ F test/e_createtable.test d4c6059d44dcd4b636de9aae322766062b471844
 F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e
 F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412
 F test/e_dropview.test 21ce09c361227ddbc9819a5608ee2700c276bdd5
-F test/e_expr.test 146deba180273d19e3bf9f6b45f4e50094c64c7ec4756ea72f79dda25818eb17
+F test/e_expr.test ca8896601ade1e27c6559614c7f32c63d44636fdfa720436a160f09b8bf66c89
 F test/e_fkey.test dcdc6ad26b1d4f07636208de4c1c22aae7c0597a685a6c10fe6da91f3191dd96
 F test/e_fts3.test 8cf40550bb088a6aa187c818c00fabe26ef82900a4cd5c66b427ccafe28bedaa
 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1b2de41453ac33de82f9cd6cbb92eee4fe184fb282c27e5efa5243c8cb239630
-R e0e8cf7279386534b02109d8fa18ad99
+P 5b2002f3df1902aaa571a0efd01ab8bae7f4d37ac4819cc51595277f4de93433
+R 219c5c95f432e38325a9c8586b5b2761
 U drh
-Z 86920861ac015f347841db4c53c64a7b
+Z e4dde272cb6c1ca1520bca8de194c744
diff --git a/manifest.uuid b/manifest.uuid
index 0ce91438a9..e0163948ed 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5b2002f3df1902aaa571a0efd01ab8bae7f4d37ac4819cc51595277f4de93433
\ No newline at end of file
+7f2bd4ff45fba29528c18cac6da983bd9b164303525d3965056f5b40f85dc83f
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index f3326d02e7..b482fafc05 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3076,7 +3076,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){
     const char *z = pExpr->u.zToken;
     assert( z!=0 );
     c = sqlite3DecOrHexToI64(z, &value);
-    if( c==1 || (c==2 && !negFlag) || (negFlag && value==SMALLEST_INT64)){
+    if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){
 #ifdef SQLITE_OMIT_FLOATING_POINT
       sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z);
 #else
@@ -3090,7 +3090,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){
       }
 #endif
     }else{
-      if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; }
+      if( negFlag ){ value = c==3 ? SMALLEST_INT64 : -value; }
       sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64);
     }
   }
diff --git a/src/main.c b/src/main.c
index 3942ed9465..6f697ffb39 100644
--- a/src/main.c
+++ b/src/main.c
@@ -3942,7 +3942,7 @@ sqlite3_int64 sqlite3_uri_int64(
 ){
   const char *z = sqlite3_uri_parameter(zFilename, zParam);
   sqlite3_int64 v;
-  if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){
+  if( z && sqlite3DecOrHexToI64(z, &v)==0 ){
     bDflt = v;
   }
   return bDflt;
diff --git a/src/util.c b/src/util.c
index ebcc5bec6e..fbe3714b77 100644
--- a/src/util.c
+++ b/src/util.c
@@ -557,16 +557,12 @@ static int compare2pow63(const char *zNum, int incr){
 ** Convert zNum to a 64-bit signed integer.  zNum must be decimal. This
 ** routine does *not* accept hexadecimal notation.
 **
-** If the zNum value is representable as a 64-bit twos-complement 
-** integer, then write that value into *pNum and return 0.
+** Returns:
 **
-** If zNum is exactly 9223372036854775808, return 2.  This special
-** case is broken out because while 9223372036854775808 cannot be a 
-** signed 64-bit integer, its negative -9223372036854775808 can be.
-**
-** If zNum is too big for a 64-bit integer and is not
-** 9223372036854775808  or if zNum contains any non-numeric text,
-** then return 1.
+**     0    Successful transformation.  Fits in a 64-bit signed integer.
+**     1    Excess text after the integer value
+**     2    Integer too large for a 64-bit signed integer or is malformed
+**     3    Special case of 9223372036854775808
 **
 ** length is the number of bytes in the string (bytes, not characters).
 ** The string is not necessarily zero-terminated.  The encoding is
@@ -579,6 +575,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   int i;
   int c = 0;
   int nonNum = 0;  /* True if input contains UTF16 with high byte non-zero */
+  int rc;          /* Baseline return code */
   const char *zStart;
   const char *zEnd = zNum + length;
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
@@ -618,31 +615,35 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   testcase( i==20 );
   if( &zNum[i]<zEnd              /* Extra bytes at the end */
    || (i==0 && zStart==zNum)     /* No digits */
-   || i>19*incr                  /* Too many digits */
    || nonNum                     /* UTF16 with high-order bytes non-zero */
   ){
+    rc = 1;
+  }else{
+    rc = 0;
+  }
+  if( i>19*incr ){                /* Too many digits */
     /* zNum is empty or contains non-numeric text or is longer
     ** than 19 digits (thus guaranteeing that it is too large) */
-    return 1;
+    return 2;
   }else if( i<19*incr ){
     /* Less than 19 digits, so we know that it fits in 64 bits */
     assert( u<=LARGEST_INT64 );
-    return 0;
+    return rc;
   }else{
     /* zNum is a 19-digit numbers.  Compare it against 9223372036854775808. */
     c = compare2pow63(zNum, incr);
     if( c<0 ){
       /* zNum is less than 9223372036854775808 so it fits */
       assert( u<=LARGEST_INT64 );
-      return 0;
+      return rc;
     }else if( c>0 ){
       /* zNum is greater than 9223372036854775808 so it overflows */
-      return 1;
+      return 2;
     }else{
       /* zNum is exactly 9223372036854775808.  Fits if negative.  The
       ** special case 2 overflow if positive */
       assert( u-1==LARGEST_INT64 );
-      return neg ? 0 : 2;
+      return neg ? rc : 3;
     }
   }
 }
@@ -655,8 +656,9 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
 ** Returns:
 **
 **     0    Successful transformation.  Fits in a 64-bit signed integer.
-**     1    Integer too large for a 64-bit signed integer or is malformed
-**     2    Special case of 9223372036854775808
+**     1    Excess text after the integer value
+**     2    Integer too large for a 64-bit signed integer or is malformed
+**     3    Special case of 9223372036854775808
 */
 int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
 #ifndef SQLITE_OMIT_HEX_INTEGER
@@ -670,7 +672,7 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
       u = u*16 + sqlite3HexToInt(z[k]);
     }
     memcpy(pOut, &u, 8);
-    return (z[k]==0 && k-i<=16) ? 0 : 1;
+    return (z[k]==0 && k-i<=16) ? 0 : 2;
   }else
 #endif /* SQLITE_OMIT_HEX_INTEGER */
   {
diff --git a/src/vdbe.c b/src/vdbe.c
index 37699bbba0..2273d1f5da 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -354,7 +354,7 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){
   if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){
     return 0;
   }
-  if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){
+  if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==0 ){
     return MEM_Int;
   }
   return MEM_Real;
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 345c41a72d..09ced1e567 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -597,14 +597,21 @@ int sqlite3VdbeMemRealify(Mem *pMem){
 */
 int sqlite3VdbeMemNumerify(Mem *pMem){
   if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){
+    int rc;
     assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 );
     assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
-    if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){
+    rc = sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc);
+    if( rc==0 ){
       MemSetTypeFlag(pMem, MEM_Int);
     }else{
-      pMem->u.r = sqlite3VdbeRealValue(pMem);
-      MemSetTypeFlag(pMem, MEM_Real);
-      sqlite3VdbeIntegerAffinity(pMem);
+      i64 i = pMem->u.i;
+      sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc);
+      if( rc==1 && pMem->u.r==(double)i ){
+        pMem->u.i = i;
+        MemSetTypeFlag(pMem, MEM_Int);
+      }else{
+        MemSetTypeFlag(pMem, MEM_Real);
+      }
     }
   }
   assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 );
diff --git a/test/e_expr.test b/test/e_expr.test
index 373ffef4f9..1cac31948b 100644
--- a/test/e_expr.test
+++ b/test/e_expr.test
@@ -1663,6 +1663,50 @@ do_expr_test e_expr-32.2.3 {
 do_expr_test e_expr-32.2.4 { 
   CAST(9223372036854775807 AS NUMERIC)
 } integer 9223372036854775807
+do_expr_test e_expr-32.2.5 { 
+  CAST('9223372036854775807 ' AS NUMERIC)
+} integer 9223372036854775807
+do_expr_test e_expr-32.2.6 { 
+  CAST('   9223372036854775807   ' AS NUMERIC)
+} integer 9223372036854775807
+do_expr_test e_expr-32.2.7 { 
+  CAST('  ' AS NUMERIC)
+} integer 0
+do_execsql_test e_expr-32.2.8 {
+  WITH t1(x) AS (VALUES
+     ('9000000000000000001'),
+     ('9000000000000000001x'),
+     ('9000000000000000001 '),
+     (' 9000000000000000001 '),
+     (' 9000000000000000001'),
+     (' 9000000000000000001.'),
+     ('9223372036854775807'),
+     ('9223372036854775807 '),
+     ('   9223372036854775807   '),
+     ('9223372036854775808'),
+     ('   9223372036854775808   '),
+     ('9223372036854775807.0'),
+     ('9223372036854775807e+0'),
+     ('-5.0'),
+     ('-5e+0'))
+  SELECT typeof(CAST(x AS NUMERIC)), CAST(x AS NUMERIC)||'' FROM t1;
+} [list \
+ integer 9000000000000000001 \
+ integer 9000000000000000001 \
+ integer 9000000000000000001 \
+ integer 9000000000000000001 \
+ integer 9000000000000000001 \
+ integer 9000000000000000001 \
+ integer 9223372036854775807 \
+ integer 9223372036854775807 \
+ integer 9223372036854775807 \
+ real 9.22337203685478e+18 \
+ real 9.22337203685478e+18 \
+ integer 9223372036854775807 \
+ integer 9223372036854775807 \
+ integer -5 \
+ integer -5 \
+]
 
 # EVIDENCE-OF: R-64550-29191 Note that the result from casting any
 # non-BLOB value into a BLOB and the result from casting any BLOB value

From 97397a70e4d754b851d972f080db5387a52cdd7c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Sep 2017 17:49:12 +0000
Subject: [PATCH 104/270] Small size and performance optimization in the
 bytecode engine.

FossilOrigin-Name: 3b3e32d4cd07a1d2d1708fffa47819345ae6b39205a9f548280d499c8f481d64
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c | 38 +++++++++++++++++---------------------
 3 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/manifest b/manifest
index 2342cdcc0c..cdc5d8be7a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sresolution\sof\slarge\sinteger\svalues\sin\s"CAST(x\sAS\sNUMERIC)".
-D 2017-09-20T10:47:10.365
+C Small\ssize\sand\sperformance\soptimization\sin\sthe\sbytecode\sengine.
+D 2017-09-20T17:49:12.127
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -532,7 +532,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 08f89979c4b359483acb21b044d8e47db55f071ca4326e8bfdca5ac4cb5770ac
+F src/vdbemem.c 0180295206f98131d904a900a7887d072eecd1388258bf0e7c2bcd20eabb679e
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5b2002f3df1902aaa571a0efd01ab8bae7f4d37ac4819cc51595277f4de93433
-R 219c5c95f432e38325a9c8586b5b2761
+P 7f2bd4ff45fba29528c18cac6da983bd9b164303525d3965056f5b40f85dc83f
+R 5b0871d836c521035772bd26987bd6b8
 U drh
-Z e4dde272cb6c1ca1520bca8de194c744
+Z 4278521e76ab29461c223833181d9fb1
diff --git a/manifest.uuid b/manifest.uuid
index e0163948ed..e7cb2fbd97 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7f2bd4ff45fba29528c18cac6da983bd9b164303525d3965056f5b40f85dc83f
\ No newline at end of file
+3b3e32d4cd07a1d2d1708fffa47819345ae6b39205a9f548280d499c8f481d64
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 09ced1e567..11caecbba6 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -114,7 +114,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){
   assert( (pMem->flags&MEM_RowSet)==0 );
   assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE
            || desiredEnc==SQLITE_UTF16BE );
-  if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){
+  if( pMem->enc==desiredEnc || !(pMem->flags&MEM_Str) ){
     return SQLITE_OK;
   }
   assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
@@ -209,6 +209,20 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
   return SQLITE_OK;
 }
 
+/*
+** It is already known that pMem contains an unterminated string.
+** Add the zero terminator.
+*/
+static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){
+  if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){
+    return SQLITE_NOMEM_BKPT;
+  }
+  pMem->z[pMem->n] = 0;
+  pMem->z[pMem->n+1] = 0;
+  pMem->flags |= MEM_Term;
+  return SQLITE_OK;
+}
+
 /*
 ** Change pMem so that its MEM_Str or MEM_Blob value is stored in
 ** MEM.zMalloc, where it can be safely written.
@@ -221,12 +235,8 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){
   if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){
     if( ExpandBlob(pMem) ) return SQLITE_NOMEM;
     if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){
-      if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){
-        return SQLITE_NOMEM_BKPT;
-      }
-      pMem->z[pMem->n] = 0;
-      pMem->z[pMem->n+1] = 0;
-      pMem->flags |= MEM_Term;
+      int rc = vdbeMemAddTerminator(pMem);
+      if( rc ) return rc;
     }
   }
   pMem->flags &= ~MEM_Ephem;
@@ -265,20 +275,6 @@ int sqlite3VdbeMemExpandBlob(Mem *pMem){
 }
 #endif
 
-/*
-** It is already known that pMem contains an unterminated string.
-** Add the zero terminator.
-*/
-static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){
-  if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){
-    return SQLITE_NOMEM_BKPT;
-  }
-  pMem->z[pMem->n] = 0;
-  pMem->z[pMem->n+1] = 0;
-  pMem->flags |= MEM_Term;
-  return SQLITE_OK;
-}
-
 /*
 ** Make sure the given Mem is \u0000 terminated.
 */

From 63d1632f1ef1856d4006b249efd4282aa0a77d04 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Sep 2017 18:07:50 +0000
Subject: [PATCH 105/270] The BLOB returned by sqlite3VdbeMemFromBtree() does
 not need to be zero-terminated.

FossilOrigin-Name: e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  6 ++----
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index cdc5d8be7a..9419550ac9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\ssize\sand\sperformance\soptimization\sin\sthe\sbytecode\sengine.
-D 2017-09-20T17:49:12.127
+C The\sBLOB\sreturned\sby\ssqlite3VdbeMemFromBtree()\sdoes\snot\sneed\sto\sbe\nzero-terminated.
+D 2017-09-20T18:07:50.056
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -532,7 +532,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 0180295206f98131d904a900a7887d072eecd1388258bf0e7c2bcd20eabb679e
+F src/vdbemem.c 4996dd14244a7757a9b8b085c03825e4b7611865e73e55df248b4f0836dcf9bf
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7f2bd4ff45fba29528c18cac6da983bd9b164303525d3965056f5b40f85dc83f
-R 5b0871d836c521035772bd26987bd6b8
+P 3b3e32d4cd07a1d2d1708fffa47819345ae6b39205a9f548280d499c8f481d64
+R bc412d1d9fd3e512c9d7b13d3be8fe98
 U drh
-Z 4278521e76ab29461c223833181d9fb1
+Z 372454472847a80abeaf6ba3b240911a
diff --git a/manifest.uuid b/manifest.uuid
index e7cb2fbd97..05e0da2a72 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3b3e32d4cd07a1d2d1708fffa47819345ae6b39205a9f548280d499c8f481d64
\ No newline at end of file
+e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 11caecbba6..4437de556a 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1012,12 +1012,10 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize(
 ){
   int rc;
   pMem->flags = MEM_Null;
-  if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){
+  if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt)) ){
     rc = sqlite3BtreePayload(pCur, offset, amt, pMem->z);
     if( rc==SQLITE_OK ){
-      pMem->z[amt] = 0;
-      pMem->z[amt+1] = 0;
-      pMem->flags = MEM_Blob|MEM_Term;
+      pMem->flags = MEM_Blob;
       pMem->n = (int)amt;
     }else{
       sqlite3VdbeMemRelease(pMem);

From 762dffa54ada2436128fa7d00c110467bbfc1705 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Sep 2017 18:47:51 +0000
Subject: [PATCH 106/270] Size and performance optimization on
 sqlite3VdbeMemGrow().

FossilOrigin-Name: 4b3f7eacb862fbb5b75cf50b72fb60dfbd0acb5818e9f83383a6a63c8bbeacdf
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  5 +++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 9419550ac9..189711f25b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sBLOB\sreturned\sby\ssqlite3VdbeMemFromBtree()\sdoes\snot\sneed\sto\sbe\nzero-terminated.
-D 2017-09-20T18:07:50.056
+C Size\sand\sperformance\soptimization\son\ssqlite3VdbeMemGrow().
+D 2017-09-20T18:47:51.074
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -532,7 +532,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 4996dd14244a7757a9b8b085c03825e4b7611865e73e55df248b4f0836dcf9bf
+F src/vdbemem.c 9496f08e196c536397039783cdfda4ad2bc944d84ccad93df03e1ed3635534d6
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3b3e32d4cd07a1d2d1708fffa47819345ae6b39205a9f548280d499c8f481d64
-R bc412d1d9fd3e512c9d7b13d3be8fe98
+P e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
+R 687f0aa7f9a7f38a700d64fc554d13e9
 U drh
-Z 372454472847a80abeaf6ba3b240911a
+Z 9d62175f9630fe98b6fce9be22a6837d
diff --git a/manifest.uuid b/manifest.uuid
index 05e0da2a72..cc3059d0cd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
\ No newline at end of file
+4b3f7eacb862fbb5b75cf50b72fb60dfbd0acb5818e9f83383a6a63c8bbeacdf
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 4437de556a..403ad889cd 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -155,7 +155,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
   assert( pMem->szMalloc==0
        || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) );
   if( n<32 ) n = 32;
-  if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){
+  if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){
     pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n);
     bPreserve = 0;
   }else{
@@ -171,7 +171,8 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
     pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc);
   }
 
-  if( bPreserve && pMem->z && ALWAYS(pMem->z!=pMem->zMalloc) ){
+  if( bPreserve && pMem->z ){
+    assert( pMem->z!=pMem->zMalloc );
     memcpy(pMem->zMalloc, pMem->z, pMem->n);
   }
   if( (pMem->flags&MEM_Dyn)!=0 ){

From b5c1063ab7f561f389256c05e0275ef3fcfbc11b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 00:49:15 +0000
Subject: [PATCH 107/270] Fix the rendering of the P4_INTARRAY argument to the
 OP_IntegrityCk opcode in the output of EXPLAIN.

FossilOrigin-Name: adc12c83dda8ba93ca220bbff649f763058b9440968ae463621f0cb7de8889cf
---
 manifest          | 18 +++++++++---------
 manifest.uuid     |  2 +-
 src/pragma.c      |  6 +++---
 src/vdbe.c        |  4 ++--
 src/vdbeaux.c     |  2 +-
 test/pragma4.test | 19 +++++++++++++++++++
 6 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 189711f25b..46a3a70a98 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Size\sand\sperformance\soptimization\son\ssqlite3VdbeMemGrow().
-D 2017-09-20T18:47:51.074
+C Fix\sthe\srendering\sof\sthe\sP4_INTARRAY\sargument\sto\sthe\sOP_IntegrityCk\sopcode\nin\sthe\soutput\sof\sEXPLAIN.
+D 2017-09-21T00:49:15.047
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -451,7 +451,7 @@ F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
-F src/pragma.c 88d99aa4752894aaf4102eefd09b5e497f4277661aff69f8fff61f8611f19d14
+F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
@@ -526,11 +526,11 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 2664b89eb243c24bd8dd66a7a13b04b5c83f41897dc80266ad13fed4bb09966c
+F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 3be977a032e54fe46cb1b1f3ba62158438b0cc93e091f6feca7742d20dad3203
+F src/vdbeaux.c 831a77aaa7aa43005f1c9bf3e9eb6506f4865e1cf99943ccdcd3be5d2dd8a3c7
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 9496f08e196c536397039783cdfda4ad2bc944d84ccad93df03e1ed3635534d6
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1101,7 +1101,7 @@ F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0e
 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
-F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264
+F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b335c
 F test/pragma5.test fd517f42ee847e126afbbbd9fd0fb9e5a4a61a962496a350adb8a22583fbdc37
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
-R 687f0aa7f9a7f38a700d64fc554d13e9
+P 4b3f7eacb862fbb5b75cf50b72fb60dfbd0acb5818e9f83383a6a63c8bbeacdf
+R 6842bc7ed52b90693780a367a8597f47
 U drh
-Z 9d62175f9630fe98b6fce9be22a6837d
+Z a3512caea006d13b8a10416043a6f9b4
diff --git a/manifest.uuid b/manifest.uuid
index cc3059d0cd..43a4f7bc69 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4b3f7eacb862fbb5b75cf50b72fb60dfbd0acb5818e9f83383a6a63c8bbeacdf
\ No newline at end of file
+adc12c83dda8ba93ca220bbff649f763058b9440968ae463621f0cb7de8889cf
\ No newline at end of file
diff --git a/src/pragma.c b/src/pragma.c
index 29c83480fc..918b1d8131 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1515,12 +1515,12 @@ void sqlite3Pragma(
       for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){
         Table *pTab = sqliteHashData(x);
         Index *pIdx;
-        if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum;
+        if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum;
         for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-          aRoot[cnt++] = pIdx->tnum;
+          aRoot[++cnt] = pIdx->tnum;
         }
       }
-      aRoot[cnt] = 0;
+      aRoot[0] = cnt;
 
       /* Make sure sufficient number of registers have been allocated */
       pParse->nMem = MAX( pParse->nMem, 8+mxIdx );
diff --git a/src/vdbe.c b/src/vdbe.c
index 2273d1f5da..9687170bec 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -5673,7 +5673,7 @@ case OP_IntegrityCk: {
   nRoot = pOp->p2;
   aRoot = pOp->p4.ai;
   assert( nRoot>0 );
-  assert( aRoot[nRoot]==0 );
+  assert( aRoot[0]==nRoot );
   assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pnErr = &aMem[pOp->p3];
   assert( (pnErr->flags & MEM_Int)!=0 );
@@ -5681,7 +5681,7 @@ case OP_IntegrityCk: {
   pIn1 = &aMem[pOp->p1];
   assert( pOp->p5<db->nDb );
   assert( DbMaskTest(p->btreeMask, pOp->p5) );
-  z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot,
+  z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, &aRoot[1], nRoot,
                                  (int)pnErr->u.i+1, &nErr);
   sqlite3VdbeMemSetNull(pIn1);
   if( nErr==0 ){
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index d426e86732..9046922c39 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1396,7 +1396,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){
       int *ai = pOp->p4.ai;
       int n = ai[0];   /* The first element of an INTARRAY is always the
                        ** count of the number of elements to follow */
-      for(i=1; i<n; i++){
+      for(i=1; i<=n; i++){
         sqlite3XPrintf(&x, ",%d", ai[i]);
       }
       zTemp[0] = '[';
diff --git a/test/pragma4.test b/test/pragma4.test
index 9a89b247de..d3ded75443 100644
--- a/test/pragma4.test
+++ b/test/pragma4.test
@@ -80,5 +80,24 @@ foreach {tn sql} {
   do_pragma_ncol_test 1.$tn.1 $sql 0
 }
 
+# EXPLAIN on a PRAGMA integrity_check.
+# Verify that that P4_INTARRAY argument to OP_IntegrityCk is rendered
+# correctly.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_test pragma4-2.100 {
+  db eval {
+    PRAGMA page_size=512;
+    CREATE TABLE t1(x);
+    WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<10000)
+    INSERT INTO t1(x) SELECT zeroblob(300) FROM c;
+    CREATE TABLE t2(y);
+    DROP TABLE t1;
+  }
+  string map {\[ x \] x \173 {} \175 {}} \
+    [db eval {EXPLAIN PRAGMA integrity_check}]
+} {/ IntegrityCk 2 2 1 x[0-9]+,1x /}
 
 finish_test

From c07df4c3b9df4acd8a98f74e592564b8bf6f76af Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 01:04:30 +0000
Subject: [PATCH 108/270] Revert one performance improvement changes from
 check-in [3b3e32d4cd07] as it was causing a reference to an uninitialized
 value.

FossilOrigin-Name: f8b1c64d3eeb8413ca149f34cc00f9154a5446d06ad33bbffa69118e6110f81d
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 46a3a70a98..3992ec18ab 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\srendering\sof\sthe\sP4_INTARRAY\sargument\sto\sthe\sOP_IntegrityCk\sopcode\nin\sthe\soutput\sof\sEXPLAIN.
-D 2017-09-21T00:49:15.047
+C Revert\sone\sperformance\simprovement\schanges\sfrom\scheck-in\s[3b3e32d4cd07]\sas\nit\swas\scausing\sa\sreference\sto\san\suninitialized\svalue.
+D 2017-09-21T01:04:30.271
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -532,7 +532,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 831a77aaa7aa43005f1c9bf3e9eb6506f4865e1cf99943ccdcd3be5d2dd8a3c7
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 9496f08e196c536397039783cdfda4ad2bc944d84ccad93df03e1ed3635534d6
+F src/vdbemem.c 043f9fdbb19d4857d5ac9c1ff60b972da9397e51c1a3d5ff43e8b6b4ae552aaf
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4b3f7eacb862fbb5b75cf50b72fb60dfbd0acb5818e9f83383a6a63c8bbeacdf
-R 6842bc7ed52b90693780a367a8597f47
+P adc12c83dda8ba93ca220bbff649f763058b9440968ae463621f0cb7de8889cf
+R 74e9a42e62567d447ce10652f9c93fed
 U drh
-Z a3512caea006d13b8a10416043a6f9b4
+Z 07fc711a459108ff2c2431795ab36ea3
diff --git a/manifest.uuid b/manifest.uuid
index 43a4f7bc69..4aaeed6a9d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-adc12c83dda8ba93ca220bbff649f763058b9440968ae463621f0cb7de8889cf
\ No newline at end of file
+f8b1c64d3eeb8413ca149f34cc00f9154a5446d06ad33bbffa69118e6110f81d
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 403ad889cd..8447ab378d 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -114,7 +114,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){
   assert( (pMem->flags&MEM_RowSet)==0 );
   assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE
            || desiredEnc==SQLITE_UTF16BE );
-  if( pMem->enc==desiredEnc || !(pMem->flags&MEM_Str) ){
+  if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){
     return SQLITE_OK;
   }
   assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );

From 19be1b631b003622523929a3587ea200975ac6d7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 10:24:10 +0000
Subject: [PATCH 109/270] Fix a bug in tool/mksourceid.c: The mksourceid.c
 program was incorrectly including the "# Remove this line" line of the
 Fossil-generated "manifest" file in the SHA3 hash.  That means that all
 SQLITE_SOURCE_IDs for trunk versions going back to check-in [30966d56]
 (2017-08-22) are incorrect.

FossilOrigin-Name: 65765222ef6f4e80c05a24994fcee145f5fbc0ad35c9fab3d75492964b3eb187
---
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 tool/mksourceid.c | 13 +++++++------
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 3992ec18ab..3f614bb807 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Revert\sone\sperformance\simprovement\schanges\sfrom\scheck-in\s[3b3e32d4cd07]\sas\nit\swas\scausing\sa\sreference\sto\san\suninitialized\svalue.
-D 2017-09-21T01:04:30.271
+C Fix\sa\sbug\sin\stool/mksourceid.c:\s\nThe\smksourceid.c\sprogram\swas\sincorrectly\sincluding\sthe\s"#\sRemove\sthis\sline"\nline\sof\sthe\sFossil-generated\s"manifest"\sfile\sin\sthe\sSHA3\shash.\s\sThat\smeans\nthat\sall\sSQLITE_SOURCE_IDs\sfor\strunk\sversions\sgoing\sback\sto\ncheck-in\s[30966d56]\s(2017-08-22)\sare\sincorrect.
+D 2017-09-21T10:24:10.329
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1593,7 +1593,7 @@ F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
 F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb3b0b
-F tool/mksourceid.c 30966d568654a4fd962fb324753e49429b7379e1f72d2be489ade963121f5943
+F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
 F tool/mksqlite3c.tcl b258d679829a9305f5cf107b7d97b9bf23adb3773df42947fed5ef7b180dfbd9
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P adc12c83dda8ba93ca220bbff649f763058b9440968ae463621f0cb7de8889cf
-R 74e9a42e62567d447ce10652f9c93fed
+P f8b1c64d3eeb8413ca149f34cc00f9154a5446d06ad33bbffa69118e6110f81d
+R 64106870c2d3f0ac5e3f96924365cf0c
 U drh
-Z 07fc711a459108ff2c2431795ab36ea3
+Z 04dce0480829a76620b41bbfd98b4888
diff --git a/manifest.uuid b/manifest.uuid
index 4aaeed6a9d..8d5471ba5f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f8b1c64d3eeb8413ca149f34cc00f9154a5446d06ad33bbffa69118e6110f81d
\ No newline at end of file
+65765222ef6f4e80c05a24994fcee145f5fbc0ad35c9fab3d75492964b3eb187
\ No newline at end of file
diff --git a/tool/mksourceid.c b/tool/mksourceid.c
index 0e20b3c3d3..282f5c4414 100644
--- a/tool/mksourceid.c
+++ b/tool/mksourceid.c
@@ -777,9 +777,10 @@ int main(int argc, char **argv){
   FILE *in;
   int allValid = 1;
   int rc;
+  SHA3Context ctx;
   char zDate[50];
   char zHash[100];
-  char zLine[1000];
+  char zLine[20000];
 
   for(i=1; i<argc; i++){
     const char *z = argv[i];
@@ -805,7 +806,11 @@ int main(int argc, char **argv){
     fprintf(stderr, "cannot open \"%s\" for reading\n", zManifest);
     exit(1);
   }
+  SHA3Init(&ctx, 256);
   while( fgets(zLine, sizeof(zLine), in) ){
+    if( strncmp(zLine,"# Remove this line", 18)!=0 ){
+      SHA3Update(&ctx, (unsigned char*)zLine, (unsigned)strlen(zLine));
+    }
     if( strncmp(zLine, "D 20", 4)==0 ){
       memcpy(zDate, &zLine[2], 10);
       zDate[10] = ' ';
@@ -826,8 +831,6 @@ int main(int argc, char **argv){
         allValid = 0;
         if( bVerbose ){
           printf("hash failed: %s\n", zFilename);
-        }else{
-          break;
         }
       }else if( strcmp(zHash, zMHash)!=0 ){
         allValid = 0;
@@ -835,14 +838,12 @@ int main(int argc, char **argv){
           printf("wrong hash: %s\n", zFilename);
           printf("... expected: %s\n", zMHash);
           printf("... got:      %s\n", zHash);
-        }else{
-          break;
         }
       }
     }
   }
   fclose(in);
-  sha3sum_file(zManifest, 256, zHash);
+  DigestToBase16(SHA3Final(&ctx), zHash, 256/8);
   if( !allValid ){
     printf("%s %.60salt1\n", zDate, zHash);
   }else{

From 9616805728717e81f3c459a3b26f2b53806c15bc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 13:11:47 +0000
Subject: [PATCH 110/270] Fix a potential use of an uninitialized pointer in
 RTree following an OOM error.

FossilOrigin-Name: fd4ec0cdbd84f3333dd4c7a4236491bce6b9ab21fb2c088751ca1279b31bd864
---
 ext/rtree/rtree.c |  2 +-
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index 40a8676d7b..32b9b3027a 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -2853,7 +2853,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){
   int rc;                         /* Return code */
   RtreeNode *pLeaf = 0;           /* Leaf node containing record iDelete */
   int iCell;                      /* Index of iDelete cell in pLeaf */
-  RtreeNode *pRoot;               /* Root node of rtree structure */
+  RtreeNode *pRoot = 0;           /* Root node of rtree structure */
 
 
   /* Obtain a reference to the root node to initialize Rtree.iDepth */
diff --git a/manifest b/manifest
index 3f614bb807..77b149ba34 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbug\sin\stool/mksourceid.c:\s\nThe\smksourceid.c\sprogram\swas\sincorrectly\sincluding\sthe\s"#\sRemove\sthis\sline"\nline\sof\sthe\sFossil-generated\s"manifest"\sfile\sin\sthe\sSHA3\shash.\s\sThat\smeans\nthat\sall\sSQLITE_SOURCE_IDs\sfor\strunk\sversions\sgoing\sback\sto\ncheck-in\s[30966d56]\s(2017-08-22)\sare\sincorrect.
-D 2017-09-21T10:24:10.329
+C Fix\sa\spotential\suse\sof\san\suninitialized\spointer\sin\sRTree\sfollowing\san\sOOM\nerror.
+D 2017-09-21T13:11:47.775
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -326,7 +326,7 @@ F ext/rbu/sqlite3rbu.c a1a303de8b90f987ef63bf9cef57f5d7dd7983a9e8aed3775a759d87a
 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c cf84d52958a7ec6a506f1711e119db847ed6bb5dedde78a58e97503287afcda1
+F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 4fdd60ae034e43f2fefc26492032d02e742e8b14d468b7c51d95a1e2fa47cf00
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f8b1c64d3eeb8413ca149f34cc00f9154a5446d06ad33bbffa69118e6110f81d
-R 64106870c2d3f0ac5e3f96924365cf0c
+P 65765222ef6f4e80c05a24994fcee145f5fbc0ad35c9fab3d75492964b3eb187
+R 02da5e87646960e2ec34004109ee9fa5
 U drh
-Z 04dce0480829a76620b41bbfd98b4888
+Z 12b5a29a0408d0f6f2c00a6a52186774
diff --git a/manifest.uuid b/manifest.uuid
index 8d5471ba5f..5ad688b147 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-65765222ef6f4e80c05a24994fcee145f5fbc0ad35c9fab3d75492964b3eb187
\ No newline at end of file
+fd4ec0cdbd84f3333dd4c7a4236491bce6b9ab21fb2c088751ca1279b31bd864
\ No newline at end of file

From 8dfde89bc655104e5e7d79f8ca22ac07c07405cd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 19:08:25 +0000
Subject: [PATCH 111/270] Fix harmless UB in the ICU extension.

FossilOrigin-Name: 68e9a840d9cfbf4988e1a68c34e809d15d0235998cf0bfa147a1ab88ea842a61
---
 ext/icu/icu.c | 14 +++++++-------
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/ext/icu/icu.c b/ext/icu/icu.c
index 7c37812d86..33c4aa76f3 100644
--- a/ext/icu/icu.c
+++ b/ext/icu/icu.c
@@ -102,15 +102,15 @@ static int icuLikeCompare(
   const uint8_t *zString,    /* The UTF-8 string to compare against */
   const UChar32 uEsc         /* The escape character */
 ){
-  static const int MATCH_ONE = (UChar32)'_';
-  static const int MATCH_ALL = (UChar32)'%';
+  static const uint32_t MATCH_ONE = (uint32_t)'_';
+  static const uint32_t MATCH_ALL = (uint32_t)'%';
 
   int prevEscape = 0;     /* True if the previous character was uEsc */
 
   while( 1 ){
 
     /* Read (and consume) the next character from the input pattern. */
-    UChar32 uPattern;
+    uint32_t uPattern;
     SQLITE_ICU_READ_UTF8(zPattern, uPattern);
     if( uPattern==0 ) break;
 
@@ -152,16 +152,16 @@ static int icuLikeCompare(
       if( *zString==0 ) return 0;
       SQLITE_ICU_SKIP_UTF8(zString);
 
-    }else if( !prevEscape && uPattern==uEsc){
+    }else if( !prevEscape && uPattern==(uint32_t)uEsc){
       /* Case 3. */
       prevEscape = 1;
 
     }else{
       /* Case 4. */
-      UChar32 uString;
+      uint32_t uString;
       SQLITE_ICU_READ_UTF8(zString, uString);
-      uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT);
-      uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT);
+      uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT);
+      uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT);
       if( uString!=uPattern ){
         return 0;
       }
diff --git a/manifest b/manifest
index 77b149ba34..2fc89bf1f8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\spotential\suse\sof\san\suninitialized\spointer\sin\sRTree\sfollowing\san\sOOM\nerror.
-D 2017-09-21T13:11:47.775
+C Fix\sharmless\sUB\sin\sthe\sICU\sextension.
+D 2017-09-21T19:08:25.780
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -209,7 +209,7 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
-F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332
+F ext/icu/icu.c 635775226d07c743c770888a9dd5175afc6e67d3e28a4032b7fedc3bcaa92e65
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/lsm1/Makefile 2951812df1c1cbc9e023af7e070876f479b3d75ce3898b3b9d00f83fecf13608
 F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 65765222ef6f4e80c05a24994fcee145f5fbc0ad35c9fab3d75492964b3eb187
-R 02da5e87646960e2ec34004109ee9fa5
+P fd4ec0cdbd84f3333dd4c7a4236491bce6b9ab21fb2c088751ca1279b31bd864
+R e47dc725154fea483b788f56f70bc385
 U drh
-Z 12b5a29a0408d0f6f2c00a6a52186774
+Z a934d9409a25854b9c4537ba2fad9aea
diff --git a/manifest.uuid b/manifest.uuid
index 5ad688b147..29b34cf006 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fd4ec0cdbd84f3333dd4c7a4236491bce6b9ab21fb2c088751ca1279b31bd864
\ No newline at end of file
+68e9a840d9cfbf4988e1a68c34e809d15d0235998cf0bfa147a1ab88ea842a61
\ No newline at end of file

From b40d9eea50ebf2091a945a40340c9e0869dc40f3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 20:03:17 +0000
Subject: [PATCH 112/270] In the Windows VFS, do not emit an SQLITE_CANTOPEN
 error log message when falling back from SQLITE_OPEN_READWRITE to
 SQLITE_OPEN_READONLY.  Wait until the open fails completely.

FossilOrigin-Name: fa3f5bcc23d9342f6df8ea15732988d637e9fa5dade85a73b05a9f66136d6964
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 2fc89bf1f8..a8b6753d68 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\sUB\sin\sthe\sICU\sextension.
-D 2017-09-21T19:08:25.780
+C In\sthe\sWindows\sVFS,\sdo\snot\semit\san\sSQLITE_CANTOPEN\serror\slog\smessage\swhen\nfalling\sback\sfrom\sSQLITE_OPEN_READWRITE\sto\sSQLITE_OPEN_READONLY.\s\sWait\suntil\nthe\sopen\sfails\scompletely.
+D 2017-09-21T20:03:17.227
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -443,7 +443,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
-F src/os_win.c 225432ab6512f63ab2f37eb76872f818b01f0483ba0bea04a7a1168be3070ea5
+F src/os_win.c 4c05a0587f38813c18bf3e9427e6be1cdf4763476bccbf335691bda3ea035e03
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd4ec0cdbd84f3333dd4c7a4236491bce6b9ab21fb2c088751ca1279b31bd864
-R e47dc725154fea483b788f56f70bc385
+P 68e9a840d9cfbf4988e1a68c34e809d15d0235998cf0bfa147a1ab88ea842a61
+R f2ed4d392aa227fcb70e42ba4d74a495
 U drh
-Z a934d9409a25854b9c4537ba2fad9aea
+Z ed7765808e7b49b8f24c258cf2e29075
diff --git a/manifest.uuid b/manifest.uuid
index 29b34cf006..ca469b6368 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-68e9a840d9cfbf4988e1a68c34e809d15d0235998cf0bfa147a1ab88ea842a61
\ No newline at end of file
+fa3f5bcc23d9342f6df8ea15732988d637e9fa5dade85a73b05a9f66136d6964
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index f7c835a522..ba8ab0ad34 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -5094,8 +5094,6 @@ static int winOpen(
            dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok"));
 
   if( h==INVALID_HANDLE_VALUE ){
-    pFile->lastErrno = lastErrno;
-    winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name);
     sqlite3_free(zConverted);
     sqlite3_free(zTmpname);
     if( isReadWrite && !isExclusive ){
@@ -5104,6 +5102,8 @@ static int winOpen(
                      ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)),
          pOutFlags);
     }else{
+      pFile->lastErrno = lastErrno;
+      winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name);
       return SQLITE_CANTOPEN_BKPT;
     }
   }

From 0e97e9a4c2d3ee1a2deae775546a93b315453a4a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Sep 2017 20:43:48 +0000
Subject: [PATCH 113/270] In the Windows VFS, when trying to open a database
 file read/write, if it fails check to see if the file exists and is read-only
 and immediately fall back to a read-only open attempt, rather than running
 the AV retry loop.

FossilOrigin-Name: 5d03c738e93d36815248991d9ed3d62297ba1bb966e602e7874410076c144f43
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 31 ++++++++++++++++++++++---------
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index a8b6753d68..2ad1b74bd0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sWindows\sVFS,\sdo\snot\semit\san\sSQLITE_CANTOPEN\serror\slog\smessage\swhen\nfalling\sback\sfrom\sSQLITE_OPEN_READWRITE\sto\sSQLITE_OPEN_READONLY.\s\sWait\suntil\nthe\sopen\sfails\scompletely.
-D 2017-09-21T20:03:17.227
+C In\sthe\sWindows\sVFS,\swhen\strying\sto\sopen\sa\sdatabase\sfile\sread/write,\sif\sit\nfails\scheck\sto\ssee\sif\sthe\sfile\sexists\sand\sis\sread-only\sand\simmediately\sfall\nback\sto\sa\sread-only\sopen\sattempt,\srather\sthan\srunning\sthe\sAV\sretry\sloop.
+D 2017-09-21T20:43:48.651
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -443,7 +443,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
-F src/os_win.c 4c05a0587f38813c18bf3e9427e6be1cdf4763476bccbf335691bda3ea035e03
+F src/os_win.c 5c802f05e706c87c6e4cc6e9527f3364c7a7178458f93dffa5e19ac2e8eef9c1
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 68e9a840d9cfbf4988e1a68c34e809d15d0235998cf0bfa147a1ab88ea842a61
-R f2ed4d392aa227fcb70e42ba4d74a495
+P fa3f5bcc23d9342f6df8ea15732988d637e9fa5dade85a73b05a9f66136d6964
+R 9677831c8749fca6b9258075623dae5d
 U drh
-Z ed7765808e7b49b8f24c258cf2e29075
+Z d25f7e57106bd66e6a2a3d4c1e5b9c3e
diff --git a/manifest.uuid b/manifest.uuid
index ca469b6368..d08d756e8b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fa3f5bcc23d9342f6df8ea15732988d637e9fa5dade85a73b05a9f66136d6964
\ No newline at end of file
+5d03c738e93d36815248991d9ed3d62297ba1bb966e602e7874410076c144f43
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index ba8ab0ad34..4c026c5c39 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -4879,6 +4879,14 @@ static int winIsDir(const void *zConverted){
   return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY);
 }
 
+/* forward reference */
+static int winAccess(
+  sqlite3_vfs *pVfs,         /* Not used on win32 */
+  const char *zFilename,     /* Name of file to check */
+  int flags,                 /* Type of test to make on this file */
+  int *pResOut               /* OUT: Result */
+);
+
 /*
 ** Open a file.
 */
@@ -5064,15 +5072,20 @@ static int winOpen(
                /* Noop */
     }
 #else
-    while( (h = osCreateFileW((LPCWSTR)zConverted,
-                              dwDesiredAccess,
-                              dwShareMode, NULL,
-                              dwCreationDisposition,
-                              dwFlagsAndAttributes,
-                              NULL))==INVALID_HANDLE_VALUE &&
-                              winRetryIoerr(&cnt, &lastErrno) ){
-               /* Noop */
-    }
+    do{
+      h = osCreateFileW((LPCWSTR)zConverted,
+                        dwDesiredAccess,
+                        dwShareMode, NULL,
+                        dwCreationDisposition,
+                        dwFlagsAndAttributes,
+                        NULL);
+      if( h!=INVALID_HANDLE_VALUE ) break;
+      if( isReadWrite ){
+        int isRO = 0;
+        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        if( rc2==SQLITE_OK && isRO ) break;
+      }
+    }while( winRetryIoerr(&cnt, &lastErrno) );
 #endif
   }
 #ifdef SQLITE_WIN32_HAS_ANSI

From 0b11bcb3e468591ae9a980271ae3b9e07a88a4c8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Sep 2017 00:24:44 +0000
Subject: [PATCH 114/270] Update the configure script so that it looks for
 tclsh8.7 ahead of tclsh8.6.

FossilOrigin-Name: 0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
---
 configure     |  2 +-
 configure.ac  |  2 +-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/configure b/configure
index ed9ed5ffd4..64b95e912c 100755
--- a/configure
+++ b/configure
@@ -10304,7 +10304,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 #
-for ac_prog in tclsh8.6 tclsh8.5 tclsh
+for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
diff --git a/configure.ac b/configure.ac
index 55ab0673ee..ace5d5d667 100644
--- a/configure.ac
+++ b/configure.ac
@@ -120,7 +120,7 @@ USE_AMALGAMATION=1
 # if not, then we fall back to plain tclsh.
 # TODO: try other versions before falling back?
 # 
-AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none)
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
diff --git a/manifest b/manifest
index 2ad1b74bd0..c6c2dbee56 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sWindows\sVFS,\swhen\strying\sto\sopen\sa\sdatabase\sfile\sread/write,\sif\sit\nfails\scheck\sto\ssee\sif\sthe\sfile\sexists\sand\sis\sread-only\sand\simmediately\sfall\nback\sto\sa\sread-only\sopen\sattempt,\srather\sthan\srunning\sthe\sAV\sretry\sloop.
-D 2017-09-21T20:43:48.651
+C Update\sthe\sconfigure\sscript\sso\sthat\sit\slooks\sfor\stclsh8.7\sahead\sof\stclsh8.6.
+D 2017-09-22T00:24:44.674
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure e691ad9b505f1f47bc5d99be9e1d49b1be9037e9cb3821c9b14c63c3d413d055 x
-F configure.ac bb85c1c53e952c8c7078a2f147eba613e0128b8b6e7780d64758d8fb29bcc695
+F configure e9dbb65b61c36bb9622225de254b768d4816749ff4cb4d71307bb067095aceec x
+F configure.ac 369ebae6c04d9d2de5064e21d300f2f42f2fbf13235cabff9d1a54f2b2c4d05d
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fa3f5bcc23d9342f6df8ea15732988d637e9fa5dade85a73b05a9f66136d6964
-R 9677831c8749fca6b9258075623dae5d
+P 5d03c738e93d36815248991d9ed3d62297ba1bb966e602e7874410076c144f43
+R af25328b1412b5532988e49269e8bcab
 U drh
-Z d25f7e57106bd66e6a2a3d4c1e5b9c3e
+Z d8a9f78bca4ad7573c0a5dc3ae2ffff1
diff --git a/manifest.uuid b/manifest.uuid
index d08d756e8b..d2fde797c5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5d03c738e93d36815248991d9ed3d62297ba1bb966e602e7874410076c144f43
\ No newline at end of file
+0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
\ No newline at end of file

From 7393c7424e14ab4acdc188a93b84d5bfaefa954b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Sep 2017 11:09:09 +0000
Subject: [PATCH 115/270] Cherrypick [ec37ad6d08] into this branch. With this
 patch, if SQLITE_SHARED_MAPPING is defined at build-time SQLite will use a
 single memory mapping for multiple connections to the same database file
 within a single process.

FossilOrigin-Name: c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264
---
 manifest      | 13 +++++++------
 manifest.uuid |  2 +-
 src/os_unix.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index b3429762a7..861009c901 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-09-22T10:49:03.297
+C Cherrypick\s[ec37ad6d08]\sinto\sthis\sbranch.\sWith\sthis\spatch,\sif\nSQLITE_SHARED_MAPPING\sis\sdefined\sat\sbuild-time\sSQLite\swill\suse\sa\ssingle\smemory\nmapping\sfor\smultiple\sconnections\sto\sthe\ssame\sdatabase\sfile\swithin\sa\ssingle\nprocess.
+D 2017-09-22T11:09:09.656
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -446,7 +446,7 @@ F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
+F src/os_unix.c 1b2203f30ec4ecef66b0f9b714432edaffe0f89f189af7c6b1db3e2cb3a52d24
 F src/os_win.c 5c802f05e706c87c6e4cc6e9527f3364c7a7178458f93dffa5e19ac2e8eef9c1
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 84e90e0acee15766840f89b8be6f14322de16396d9229c494fec6c1cc03a69cf
@@ -1666,7 +1666,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d53108e76307fb24ef1d8f75fe82c180f5588ad8e65eeceace92f4cecadfa452 0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
-R 5b090dbbf85dc5792098185268ac7c8f
+P 307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b
+Q +ec37ad6d08362f4c9faad9b629c0fa23f5864ff6ad7f4cbed93a25d5f7b815d8
+R a7069c27bdffb405c8574ad991612563
 U dan
-Z 51723e9e7fe6c543a37219e2d01ad740
+Z 9a1235c2d010c6dd3774c97b07deb6f2
diff --git a/manifest.uuid b/manifest.uuid
index f7b69a87af..da8f0bb88e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b
\ No newline at end of file
+c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 4445104dd6..0e444e61f2 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -1115,6 +1115,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -1249,6 +1253,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       closePendingFds(pFile);
       if( pInode->pPrev ){
         assert( pInode->pPrev->pNext==pInode );
@@ -2055,6 +2066,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -3874,6 +3893,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -4774,6 +4796,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -4905,6 +4930,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -5342,6 +5389,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();

From 24ddadfa3bd7fc679474b296047a73927e84e736 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Sep 2017 12:52:31 +0000
Subject: [PATCH 116/270] Partial backout of check-in [e0af9a904076].  It turns
 out we do need some extra space at the end of the record blob as an overrun
 area to use when decoding a maliciously malformed record.

FossilOrigin-Name: 403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
---
 manifest      | 13 +++++++------
 manifest.uuid |  2 +-
 src/vdbemem.c |  3 ++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index c6c2dbee56..35282026af 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sconfigure\sscript\sso\sthat\sit\slooks\sfor\stclsh8.7\sahead\sof\stclsh8.6.
-D 2017-09-22T00:24:44.674
+C Partial\sbackout\sof\scheck-in\s[e0af9a904076].\s\sIt\sturns\sout\swe\sdo\sneed\ssome\nextra\sspace\sat\sthe\send\sof\sthe\srecord\sblob\sas\san\soverrun\sarea\sto\suse\swhen\ndecoding\sa\smaliciously\smalformed\srecord.
+D 2017-09-22T12:52:31.525
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -532,7 +532,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c 831a77aaa7aa43005f1c9bf3e9eb6506f4865e1cf99943ccdcd3be5d2dd8a3c7
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
-F src/vdbemem.c 043f9fdbb19d4857d5ac9c1ff60b972da9397e51c1a3d5ff43e8b6b4ae552aaf
+F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
@@ -1655,7 +1655,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5d03c738e93d36815248991d9ed3d62297ba1bb966e602e7874410076c144f43
-R af25328b1412b5532988e49269e8bcab
+P 0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
+Q -e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
+R 2559539454c3155961a04b42be8e4a30
 U drh
-Z d8a9f78bca4ad7573c0a5dc3ae2ffff1
+Z df9a1372bda63aba20d2b49e01e54f8d
diff --git a/manifest.uuid b/manifest.uuid
index d2fde797c5..d0bf777cd9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
\ No newline at end of file
+403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 8447ab378d..3c07f5a1a3 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1013,9 +1013,10 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize(
 ){
   int rc;
   pMem->flags = MEM_Null;
-  if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt)) ){
+  if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){
     rc = sqlite3BtreePayload(pCur, offset, amt, pMem->z);
     if( rc==SQLITE_OK ){
+      pMem->z[amt] = 0;   /* Overrun area used when reading malformed records */
       pMem->flags = MEM_Blob;
       pMem->n = (int)amt;
     }else{

From 3314062b9d710f64e03638d1a6d48e9bb3ae7d98 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 22 Sep 2017 16:23:23 +0000
Subject: [PATCH 117/270] Use the updated Win32 VFS semantics for winOpen from
 check-in [5d03c738e9] for WinRT, et al, as well.

FossilOrigin-Name: 2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
---
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 src/os_win.c  | 44 +++++++++++++++++++++++++++-----------------
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/manifest b/manifest
index 35282026af..1938515870 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Partial\sbackout\sof\scheck-in\s[e0af9a904076].\s\sIt\sturns\sout\swe\sdo\sneed\ssome\nextra\sspace\sat\sthe\send\sof\sthe\srecord\sblob\sas\san\soverrun\sarea\sto\suse\swhen\ndecoding\sa\smaliciously\smalformed\srecord.
-D 2017-09-22T12:52:31.525
+C Use\sthe\supdated\sWin32\sVFS\ssemantics\sfor\swinOpen\sfrom\scheck-in\s[5d03c738e9]\sfor\sWinRT,\set\sal,\sas\swell.
+D 2017-09-22T16:23:23.946
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -443,7 +443,7 @@ F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
-F src/os_win.c 5c802f05e706c87c6e4cc6e9527f3364c7a7178458f93dffa5e19ac2e8eef9c1
+F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1655,8 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0a12915b373cd0491a58d8f7a645711c620c70efced623e6b40aa01f23284157
-Q -e0af9a9040768adf8bba42a8780adeb6304bc442afb1f35d239d019db1624f40
-R 2559539454c3155961a04b42be8e4a30
-U drh
-Z df9a1372bda63aba20d2b49e01e54f8d
+P 403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
+R 7f08f0df04c939d1dd5a187c9ae3cc49
+U mistachkin
+Z 321f82ff3326ba269549cfe661409f34
diff --git a/manifest.uuid b/manifest.uuid
index d0bf777cd9..04153e1aac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
\ No newline at end of file
+2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 4c026c5c39..245a86045b 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -5063,14 +5063,19 @@ static int winOpen(
     extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS;
     extendedParameters.lpSecurityAttributes = NULL;
     extendedParameters.hTemplateFile = NULL;
-    while( (h = osCreateFile2((LPCWSTR)zConverted,
-                              dwDesiredAccess,
-                              dwShareMode,
-                              dwCreationDisposition,
-                              &extendedParameters))==INVALID_HANDLE_VALUE &&
-                              winRetryIoerr(&cnt, &lastErrno) ){
-               /* Noop */
-    }
+    do{
+      h = osCreateFile2((LPCWSTR)zConverted,
+                        dwDesiredAccess,
+                        dwShareMode,
+                        dwCreationDisposition,
+                        &extendedParameters);
+      if( h!=INVALID_HANDLE_VALUE ) break;
+      if( isReadWrite ){
+        int isRO = 0;
+        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        if( rc2==SQLITE_OK && isRO ) break;
+      }
+    }while( winRetryIoerr(&cnt, &lastErrno) );
 #else
     do{
       h = osCreateFileW((LPCWSTR)zConverted,
@@ -5090,15 +5095,20 @@ static int winOpen(
   }
 #ifdef SQLITE_WIN32_HAS_ANSI
   else{
-    while( (h = osCreateFileA((LPCSTR)zConverted,
-                              dwDesiredAccess,
-                              dwShareMode, NULL,
-                              dwCreationDisposition,
-                              dwFlagsAndAttributes,
-                              NULL))==INVALID_HANDLE_VALUE &&
-                              winRetryIoerr(&cnt, &lastErrno) ){
-               /* Noop */
-    }
+    do{
+      h = osCreateFileA((LPCSTR)zConverted,
+                        dwDesiredAccess,
+                        dwShareMode, NULL,
+                        dwCreationDisposition,
+                        dwFlagsAndAttributes,
+                        NULL);
+      if( h!=INVALID_HANDLE_VALUE ) break;
+      if( isReadWrite ){
+        int isRO = 0;
+        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        if( rc2==SQLITE_OK && isRO ) break;
+      }
+    }while( winRetryIoerr(&cnt, &lastErrno) );
   }
 #endif
   winLogIoerr(cnt, __LINE__);

From b92d585c60bc764b6b0dd28de01b9e656a309c17 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 25 Sep 2017 09:37:37 +0000
Subject: [PATCH 118/270] Add an extra snapshot related test case.

FossilOrigin-Name: 24a95e143785bb8e12198092d13c979ec2e116fa8b55d0bf482cb473a92294d8
---
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 test/snapshot2.test | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 1938515870..01453b7f66 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sthe\supdated\sWin32\sVFS\ssemantics\sfor\swinOpen\sfrom\scheck-in\s[5d03c738e9]\sfor\sWinRT,\set\sal,\sas\swell.
-D 2017-09-22T16:23:23.946
+C Add\san\sextra\ssnapshot\srelated\stest\scase.
+D 2017-09-25T09:37:37.738
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1205,7 +1205,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5
 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2
 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b
 F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f
-F test/snapshot2.test 867652ed4a13282dce218723535fad1c7b44c3c4
+F test/snapshot2.test 925e42427e923262db63c9d7155183f889e3e99feaedec4075f659e51608344f
 F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
-R 7f08f0df04c939d1dd5a187c9ae3cc49
-U mistachkin
-Z 321f82ff3326ba269549cfe661409f34
+P 2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
+R e3a6bb3c09ba5319582a3c2241441527
+U dan
+Z 02534471970367b2995d58c98bf3758e
diff --git a/manifest.uuid b/manifest.uuid
index 04153e1aac..23a21dc119 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
\ No newline at end of file
+24a95e143785bb8e12198092d13c979ec2e116fa8b55d0bf482cb473a92294d8
\ No newline at end of file
diff --git a/test/snapshot2.test b/test/snapshot2.test
index 41e555258b..3faabd7f4c 100644
--- a/test/snapshot2.test
+++ b/test/snapshot2.test
@@ -197,4 +197,45 @@ do_test 4.7 {
   list [catch { sqlite3_snapshot_recover db aux } msg] $msg
 } {1 SQLITE_ERROR}
 
+#-------------------------------------------------------------------------
+reset_db
+sqlite3 db2 test.db 
+do_execsql_test 5.0 {
+  CREATE TABLE t2(x);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t2 VALUES('abc');
+  INSERT INTO t2 VALUES('def');
+  INSERT INTO t2 VALUES('ghi');
+} {wal}
+
+do_test 5.1 {
+  execsql { 
+    SELECT * FROM t2;
+    BEGIN;
+  } db2
+  set snap [sqlite3_snapshot_get_blob db2 main]
+  db2 eval END
+} {}
+
+do_test 5.2 {
+  execsql BEGIN db2
+  sqlite3_snapshot_open_blob db2 main $snap
+  db2 eval { SELECT * FROM t2 ; END }
+} {abc def ghi}
+
+do_test 5.3 {
+  execsql { PRAGMA wal_checkpoint = RESTART }
+  execsql BEGIN db2
+  sqlite3_snapshot_open_blob db2 main $snap
+  db2 eval { SELECT * FROM t2 ; END }
+} {abc def ghi}
+
+do_test 5.4 {
+  execsql { INSERT INTO t2 VALUES('jkl') } 
+  execsql BEGIN db2
+  list [catch { sqlite3_snapshot_open_blob db2 main $snap } msg] $msg
+} {1 SQLITE_BUSY_SNAPSHOT}
+
+
 finish_test
+

From 9325c87c2648dce3136339c3eb14bf82a1996a77 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Sep 2017 16:51:00 +0000
Subject: [PATCH 119/270] Disable tests for the LIKE optimization when ICU is
 enabled, since the LIKE optimization only works for the built-in LIKE
 operator and ICU overrides the built-in.

FossilOrigin-Name: f5ef2e1bcd2790f960c15947f87cda19bd6a6a384ad09b25f6d07d5244cfecbd
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 test/like.test |  5 ++---
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 01453b7f66..42d433896b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\sextra\ssnapshot\srelated\stest\scase.
-D 2017-09-25T09:37:37.738
+C Disable\stests\sfor\sthe\sLIKE\soptimization\swhen\sICU\sis\senabled,\ssince\sthe\nLIKE\soptimization\sonly\sworks\sfor\sthe\sbuilt-in\sLIKE\soperator\sand\sICU\soverrides\nthe\sbuilt-in.
+D 2017-09-27T16:51:00.557
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -987,7 +987,7 @@ F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c fcb38ffe3db028a3138b4818fc098359c80dc51a0d1278a91c99c554cc1abb92
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
-F test/like.test 67d7431c9b664254febce9e90fd2f47c7c75c8b38444e2a50ef9ec2776b84ca8
+F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a6
 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
 F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4
 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
-R e3a6bb3c09ba5319582a3c2241441527
-U dan
-Z 02534471970367b2995d58c98bf3758e
+P 24a95e143785bb8e12198092d13c979ec2e116fa8b55d0bf482cb473a92294d8
+R ea36f216c91269073f8e9e0d7f835975
+U drh
+Z 625452c09eb5e3262c5fc4d08882748f
diff --git a/manifest.uuid b/manifest.uuid
index 23a21dc119..dc276144d1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-24a95e143785bb8e12198092d13c979ec2e116fa8b55d0bf482cb473a92294d8
\ No newline at end of file
+f5ef2e1bcd2790f960c15947f87cda19bd6a6a384ad09b25f6d07d5244cfecbd
\ No newline at end of file
diff --git a/test/like.test b/test/like.test
index 1702dde714..e5324aee25 100644
--- a/test/like.test
+++ b/test/like.test
@@ -1048,6 +1048,7 @@ ifcapable !icu {
   } {1}
 }
 
+ifcapable !icu {
 # As of 2017-07-27 (3.21.0) the LIKE optimization works with ESCAPE as
 # long as the ESCAPE is a single-byte literal.
 #
@@ -1092,8 +1093,6 @@ do_execsql_test like-15.121 {
   EXPLAIN QUERY PLAN
   SELECT y FROM t15 WHERE x LIKE '/%bc%' ESCAPE '/';
 } {/SEARCH/}
-
-
-
+}
 
 finish_test

From 488e619192606c2d469c241e9b056c4924cd3c51 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 00:01:36 +0000
Subject: [PATCH 120/270] In two places, change the magic number -1 to its
 proper symbol XN_ROWID.

FossilOrigin-Name: 80277d2fc9b76fe41e345d00952da1528e69884f25911cf6e4f78b09ff778421
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/where.c   |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 42d433896b..2136b1cc22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\stests\sfor\sthe\sLIKE\soptimization\swhen\sICU\sis\senabled,\ssince\sthe\nLIKE\soptimization\sonly\sworks\sfor\sthe\sbuilt-in\sLIKE\soperator\sand\sICU\soverrides\nthe\sbuilt-in.
-D 2017-09-27T16:51:00.557
+C In\stwo\splaces,\schange\sthe\smagic\snumber\s-1\sto\sits\sproper\ssymbol\sXN_ROWID.
+D 2017-09-28T00:01:36.686
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -540,7 +540,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 21eea981920a13fd3c0ac3d6c128d0a34b22cbec064e4f0603375fe1ffe26ca6
+F src/where.c da814d09ab1e5400221e74f8c5fd2e270bc2bd88f8879c90b945ae969440760d
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c 4953ca4e769c047d0a00a1ba9085849626b1f3a6e89f6befcf5c38fa0722acdd
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24a95e143785bb8e12198092d13c979ec2e116fa8b55d0bf482cb473a92294d8
-R ea36f216c91269073f8e9e0d7f835975
+P f5ef2e1bcd2790f960c15947f87cda19bd6a6a384ad09b25f6d07d5244cfecbd
+R c47ffb2981ef5a31e85dd8c84d5bf56c
 U drh
-Z 625452c09eb5e3262c5fc4d08882748f
+Z 7390013eff4c87ebba9e8675c4ba2e1e
diff --git a/manifest.uuid b/manifest.uuid
index dc276144d1..5317227d34 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f5ef2e1bcd2790f960c15947f87cda19bd6a6a384ad09b25f6d07d5244cfecbd
\ No newline at end of file
+80277d2fc9b76fe41e345d00952da1528e69884f25911cf6e4f78b09ff778421
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index e5fad790b5..dfb94e570e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3658,7 +3658,7 @@ static i8 wherePathSatisfiesOrderBy(
         if( pIndex ){
           iColumn = pIndex->aiColumn[j];
           revIdx = pIndex->aSortOrder[j];
-          if( iColumn==pIndex->pTable->iPKey ) iColumn = -1;
+          if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
         }else{
           iColumn = XN_ROWID;
           revIdx = 0;
@@ -3685,7 +3685,7 @@ static i8 wherePathSatisfiesOrderBy(
           testcase( wctrlFlags & WHERE_GROUPBY );
           testcase( wctrlFlags & WHERE_DISTINCTBY );
           if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
-          if( iColumn>=(-1) ){
+          if( iColumn>=XN_ROWID ){
             if( pOBExpr->op!=TK_COLUMN ) continue;
             if( pOBExpr->iTable!=iCur ) continue;
             if( pOBExpr->iColumn!=iColumn ) continue;

From db8e68b4cd5001d904d4ba93d9b241382d5642a1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 01:09:42 +0000
Subject: [PATCH 121/270] Indexes on expressions with a COLLATE clause are able
 to satisfy an ORDER BY with the same COLLATE clause.

FossilOrigin-Name: 0413001843dce7c63659d39b329ca14cdcd54f4407922f51b2fb7659572a733e
---
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 src/where.c          |  6 +++---
 test/indexexpr2.test | 22 ++++++++++++++++++++++
 4 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 2136b1cc22..586f2edb75 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\stwo\splaces,\schange\sthe\smagic\snumber\s-1\sto\sits\sproper\ssymbol\sXN_ROWID.
-D 2017-09-28T00:01:36.686
+C Indexes\son\sexpressions\swith\sa\sCOLLATE\sclause\sare\sable\sto\ssatisfy\san\sORDER\sBY\s\nwith\sthe\ssame\sCOLLATE\sclause.
+D 2017-09-28T01:09:42.223
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -540,7 +540,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c da814d09ab1e5400221e74f8c5fd2e270bc2bd88f8879c90b945ae969440760d
+F src/where.c 5b004ea313a0250ed94d064b7795831ca72b53f956e6b776b5102ebe8e43532e
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c 4953ca4e769c047d0a00a1ba9085849626b1f3a6e89f6befcf5c38fa0722acdd
@@ -946,7 +946,7 @@ F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
 F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
-F test/indexexpr2.test fdccd5c13a57af59a8e392660953dbcaacc4699c433516372cfba52994aa503a
+F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f5ef2e1bcd2790f960c15947f87cda19bd6a6a384ad09b25f6d07d5244cfecbd
-R c47ffb2981ef5a31e85dd8c84d5bf56c
+P 80277d2fc9b76fe41e345d00952da1528e69884f25911cf6e4f78b09ff778421
+R c774dcc4924296b1d5d3f0941d695b78
 U drh
-Z 7390013eff4c87ebba9e8675c4ba2e1e
+Z 3745bfdcfe1c333d46cc8a31ac4a80d4
diff --git a/manifest.uuid b/manifest.uuid
index 5317227d34..578cfcf04f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-80277d2fc9b76fe41e345d00952da1528e69884f25911cf6e4f78b09ff778421
\ No newline at end of file
+0413001843dce7c63659d39b329ca14cdcd54f4407922f51b2fb7659572a733e
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index dfb94e570e..a02923c991 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2669,7 +2669,7 @@ static int indexMightHelpWithOrderBy(
     }else if( (aColExpr = pIndex->aColExpr)!=0 ){
       for(jj=0; jj<pIndex->nKeyCol; jj++){
         if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
-        if( sqlite3ExprCompare(0, pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
+        if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
           return 1;
         }
       }
@@ -3690,8 +3690,8 @@ static i8 wherePathSatisfiesOrderBy(
             if( pOBExpr->iTable!=iCur ) continue;
             if( pOBExpr->iColumn!=iColumn ) continue;
           }else{
-            if( sqlite3ExprCompare(0,
-                  pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){
+            Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr;
+            if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){
               continue;
             }
           }
diff --git a/test/indexexpr2.test b/test/indexexpr2.test
index d8b20934e7..037db0420a 100644
--- a/test/indexexpr2.test
+++ b/test/indexexpr2.test
@@ -136,4 +136,26 @@ do_execsql_test 3.4.2 {
   .ABC 1   .ABC 3   .abc 2   .abc 4
 }
 
+do_execsql_test 3.4.3 {
+  DROP INDEX i4;
+  UPDATE t4 SET a = printf('%s%d',a,b);
+  SELECT * FROM t4 ORDER BY Substr(a,-2) COLLATE nocase;
+} {.ABC1 1 .abc2 2 .ABC3 3 .abc4 4}
+do_execsql_test 3.4.4 {
+  SELECT * FROM t4 ORDER BY Substr(a,-2) COLLATE binary;
+} {.ABC1 1 .ABC3 3 .abc2 2 .abc4 4}
+
+do_execsql_test 3.4.5 {
+  CREATE INDEX i4 ON t4( Substr(a,-2) COLLATE nocase );
+  SELECT * FROM t4 ORDER BY Substr(a,-2) COLLATE nocase;
+} {.ABC1 1 .abc2 2 .ABC3 3 .abc4 4}
+do_execsql_test 3.4.5eqp {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t4 ORDER BY Substr(a,-2) COLLATE nocase;
+} {/SCAN TABLE t4 USING INDEX i4/}
+do_execsql_test 3.4.6 {
+  SELECT * FROM t4 ORDER BY Substr(a,-2) COLLATE binary;
+} {.ABC1 1 .ABC3 3 .abc2 2 .abc4 4}
+
+
 finish_test

From 70efa84da7fab3c84cb5514093f77040e382c129 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 01:58:23 +0000
Subject: [PATCH 122/270] Add new routines to simplify dealing with collating
 sequences in expressions: sqlite3ExprNNCollSeq() and
 sqlite3ExprCollSeqMatch().

FossilOrigin-Name: 490e488ea963fe725b16212822c8608f2b6abce688931b611446bc2cbfe6b87c
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/expr.c      | 35 +++++++++++++++++++++++++++++++++--
 src/select.c    |  5 +----
 src/sqliteInt.h |  2 ++
 src/where.c     | 19 +++++++------------
 src/whereexpr.c |  7 +------
 7 files changed, 55 insertions(+), 35 deletions(-)

diff --git a/manifest b/manifest
index 586f2edb75..5a19f39844 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Indexes\son\sexpressions\swith\sa\sCOLLATE\sclause\sare\sable\sto\ssatisfy\san\sORDER\sBY\s\nwith\sthe\ssame\sCOLLATE\sclause.
-D 2017-09-28T01:09:42.223
+C Add\snew\sroutines\sto\ssimplify\sdealing\swith\scollating\ssequences\sin\sexpressions:\nsqlite3ExprNNCollSeq()\sand\ssqlite3ExprCollSeqMatch().
+D 2017-09-28T01:58:23.335
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -411,7 +411,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 82fedd57c8ce9e7dc16a003ad4cd863308787d5b5cbd0f83263b37805a56319c
+F src/expr.c 628395aea8bbf4e2851565f9dab76565e3100ca0836a337fb748834c9cec9f04
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -458,13 +458,13 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 9fa0db382f43217e207a145b8c6cec26e85cd1a42a8428ee8b3df5870dfea0f4
+F src/select.c 843ac757ad04480845ce4c2d2f742c13c8f751dec115a9a7d3dbef33c9264598
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 12aa1f626b3209ffa6a50d9d1e6b4235abd33273a0fcbfeedb66f573a68932b9
+F src/sqliteInt.h 954fed875c59d283870f13010bbbb96d8b25cf3db0683843403c36e9dfcf1bdd
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -540,10 +540,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 5b004ea313a0250ed94d064b7795831ca72b53f956e6b776b5102ebe8e43532e
+F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
-F src/whereexpr.c 4953ca4e769c047d0a00a1ba9085849626b1f3a6e89f6befcf5c38fa0722acdd
+F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 80277d2fc9b76fe41e345d00952da1528e69884f25911cf6e4f78b09ff778421
-R c774dcc4924296b1d5d3f0941d695b78
+P 0413001843dce7c63659d39b329ca14cdcd54f4407922f51b2fb7659572a733e
+R 0aa0c6a22a3ca8a1377c9aa1ffa8dd82
 U drh
-Z 3745bfdcfe1c333d46cc8a31ac4a80d4
+Z 1f5f89ecf882e16436178057d38d6ed8
diff --git a/manifest.uuid b/manifest.uuid
index 578cfcf04f..030ba54b7f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0413001843dce7c63659d39b329ca14cdcd54f4407922f51b2fb7659572a733e
\ No newline at end of file
+490e488ea963fe725b16212822c8608f2b6abce688931b611446bc2cbfe6b87c
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index b482fafc05..8282c99a4a 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -124,6 +124,11 @@ Expr *sqlite3ExprSkipCollate(Expr *pExpr){
 ** Return the collation sequence for the expression pExpr. If
 ** there is no defined collating sequence, return NULL.
 **
+** See also: sqlite3ExprNNCollSeq()
+**
+** The sqlite3ExprNNCollSeq() works the same exact that it returns the
+** default collation if pExpr has no defined collation.
+**
 ** The collating sequence might be determined by a COLLATE operator
 ** or by the presence of a column with a defined collating sequence.
 ** COLLATE operators take first precedence.  Left operands take
@@ -188,6 +193,32 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){
   return pColl;
 }
 
+/*
+** Return the collation sequence for the expression pExpr. If
+** there is no defined collating sequence, return a pointer to the
+** defautl collation sequence.
+**
+** See also: sqlite3ExprCollSeq()
+**
+** The sqlite3ExprCollSeq() routine works the same except that it
+** returns NULL if there is no defined collation.
+*/
+CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr){
+  CollSeq *p = sqlite3ExprCollSeq(pParse, pExpr);
+  if( p==0 ) p = pParse->db->pDfltColl;
+  assert( p!=0 );
+  return p;
+}
+
+/*
+** Return TRUE if the two expressions have equivalent collating sequences.
+*/
+int sqlite3ExprCollSeqMatch(Parse *pParse, Expr *pE1, Expr *pE2){
+  CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pE1);
+  CollSeq *pColl2 = sqlite3ExprNNCollSeq(pParse, pE2);
+  return sqlite3StrICmp(pColl1->zName, pColl2->zName)==0;
+}
+
 /*
 ** pExpr is an operand of a comparison operator.  aff2 is the
 ** type affinity of the other operand.  This routine returns the
@@ -1843,8 +1874,8 @@ static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){
   for(i=0; i<pGroupBy->nExpr; i++){
     Expr *p = pGroupBy->a[i].pExpr;
     if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){
-      CollSeq *pColl = sqlite3ExprCollSeq(pWalker->pParse, p);
-      if( pColl==0 || sqlite3_stricmp("BINARY", pColl->zName)==0 ){
+      CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p);
+      if( sqlite3_stricmp("BINARY", pColl->zName)==0 ){
         return WRC_Prune;
       }
     }
diff --git a/src/select.c b/src/select.c
index 971e7f5d4e..79abda90a3 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1109,10 +1109,7 @@ static KeyInfo *keyInfoFromExprList(
   if( pInfo ){
     assert( sqlite3KeyInfoIsWriteable(pInfo) );
     for(i=iStart, pItem=pList->a+iStart; i<nExpr; i++, pItem++){
-      CollSeq *pColl;
-      pColl = sqlite3ExprCollSeq(pParse, pItem->pExpr);
-      if( !pColl ) pColl = db->pDfltColl;
-      pInfo->aColl[i-iStart] = pColl;
+      pInfo->aColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr);
       pInfo->aSortOrder[i-iStart] = pItem->sortOrder;
     }
   }
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 14514007c4..3259df894f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4004,6 +4004,8 @@ int sqlite3ReadSchema(Parse *pParse);
 CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int);
 CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName);
 CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr);
+CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr);
+int sqlite3ExprCollSeqMatch(Parse*,Expr*,Expr*);
 Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int);
 Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*);
 Expr *sqlite3ExprSkipCollate(Expr*);
diff --git a/src/where.c b/src/where.c
index a02923c991..5a3d44f9ee 100644
--- a/src/where.c
+++ b/src/where.c
@@ -403,8 +403,8 @@ static int findIndexCol(
      && p->iColumn==pIdx->aiColumn[iCol]
      && p->iTable==iBase
     ){
-      CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr);
-      if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){
+      CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
+      if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
         return i;
       }
     }
@@ -3579,14 +3579,10 @@ static i8 wherePathSatisfiesOrderBy(
         if( j>=pLoop->nLTerm ) continue;
       }
       if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
-        const char *z1, *z2;
-        pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
-        if( !pColl ) pColl = db->pDfltColl;
-        z1 = pColl->zName;
-        pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr);
-        if( !pColl ) pColl = db->pDfltColl;
-        z2 = pColl->zName;
-        if( sqlite3StrICmp(z1, z2)!=0 ) continue;
+        if( sqlite3ExprCollSeqMatch(pWInfo->pParse, 
+                  pOrderBy->a[i].pExpr, pTerm->pExpr)==0 ){
+          continue;
+        }
         testcase( pTerm->pExpr->op==TK_IS );
       }
       obSat |= MASKBIT(i);
@@ -3696,8 +3692,7 @@ static i8 wherePathSatisfiesOrderBy(
             }
           }
           if( iColumn!=XN_ROWID ){
-            pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
-            if( !pColl ) pColl = db->pDfltColl;
+            pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
             if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
           }
           pLoop->u.btree.nIdxCol = j+1;
diff --git a/src/whereexpr.c b/src/whereexpr.c
index da07f56ce8..9f83a84534 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -842,7 +842,6 @@ static void exprAnalyzeOrTerm(
 static int termIsEquivalence(Parse *pParse, Expr *pExpr){
   char aff1, aff2;
   CollSeq *pColl;
-  const char *zColl1, *zColl2;
   if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0;
   if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0;
   if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0;
@@ -855,11 +854,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){
   }
   pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight);
   if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1;
-  pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
-  zColl1 = pColl ? pColl->zName : 0;
-  pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight);
-  zColl2 = pColl ? pColl->zName : 0;
-  return sqlite3_stricmp(zColl1, zColl2)==0;
+  return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight);
 }
 
 /*

From 4e6cec1ca0d927b4f62b202554fa607c7987758c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 13:47:35 +0000
Subject: [PATCH 123/270] Fix an issue introduced by check-in [4cd2a9672c59]
 (2017-03-03) that could allow a negative value in the 3rd parameter to
 memmove() when defragmentPage() is called on a btree page with a corrupted
 freeblock list.  The corruption is now detected early and results in an
 SQLITE_CORRUPT return before the memmove() is reached.

FossilOrigin-Name: 5b9ae693120fe4f7bc3b6270f35d773876f6cc8f5990e05cce0d255c54b36ae7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  3 +++
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 5a19f39844..a65a171730 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\sroutines\sto\ssimplify\sdealing\swith\scollating\ssequences\sin\sexpressions:\nsqlite3ExprNNCollSeq()\sand\ssqlite3ExprCollSeqMatch().
-D 2017-09-28T01:58:23.335
+C Fix\san\sissue\sintroduced\sby\scheck-in\s[4cd2a9672c59]\s(2017-03-03)\sthat\scould\nallow\sa\snegative\svalue\sin\sthe\s3rd\sparameter\sto\smemmove()\swhen\ndefragmentPage()\sis\scalled\son\sa\sbtree\spage\swith\sa\scorrupted\nfreeblock\slist.\s\sThe\scorruption\sis\snow\sdetected\searly\sand\sresults\sin\nan\sSQLITE_CORRUPT\sreturn\sbefore\sthe\smemmove()\sis\sreached.
+D 2017-09-28T13:47:35.240
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -401,7 +401,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 1c2b2f1714c411d7a9bc52c90d9dd7eab261261d5691ac0f67e1ced92419799c
+F src/btree.c 221bc1b836f0c386676999a7c62c8dc60455e255fab37df97eca2aa619b92f2a
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0413001843dce7c63659d39b329ca14cdcd54f4407922f51b2fb7659572a733e
-R 0aa0c6a22a3ca8a1377c9aa1ffa8dd82
+P 490e488ea963fe725b16212822c8608f2b6abce688931b611446bc2cbfe6b87c
+R 2f97a6d6fe698142318cfe94f9a0a57b
 U drh
-Z 1f5f89ecf882e16436178057d38d6ed8
+Z 00f2b9734ad71698420412023f6d2194
diff --git a/manifest.uuid b/manifest.uuid
index 030ba54b7f..567c9cdf5d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-490e488ea963fe725b16212822c8608f2b6abce688931b611446bc2cbfe6b87c
\ No newline at end of file
+5b9ae693120fe4f7bc3b6270f35d773876f6cc8f5990e05cce0d255c54b36ae7
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index a1b125dda8..7c468f35a5 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -1399,6 +1399,9 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
         int sz2 = 0;
         int sz = get2byte(&data[iFree+2]);
         int top = get2byte(&data[hdr+5]);
+        if( top>=iFree ){
+          return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        }
         if( iFree2 ){
           assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */
           sz2 = get2byte(&data[iFree2+2]);

From 3d240d21196bab3be107a4ca0a278cc5e7925f39 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 16:56:55 +0000
Subject: [PATCH 124/270] Fix over-length source code lines in select.c.  No
 logic changes.

FossilOrigin-Name: fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 54 +++++++++++++++++++++++++++++----------------------
 3 files changed, 38 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index a65a171730..f258da1133 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sissue\sintroduced\sby\scheck-in\s[4cd2a9672c59]\s(2017-03-03)\sthat\scould\nallow\sa\snegative\svalue\sin\sthe\s3rd\sparameter\sto\smemmove()\swhen\ndefragmentPage()\sis\scalled\son\sa\sbtree\spage\swith\sa\scorrupted\nfreeblock\slist.\s\sThe\scorruption\sis\snow\sdetected\searly\sand\sresults\sin\nan\sSQLITE_CORRUPT\sreturn\sbefore\sthe\smemmove()\sis\sreached.
-D 2017-09-28T13:47:35.240
+C Fix\sover-length\ssource\scode\slines\sin\sselect.c.\s\sNo\slogic\schanges.
+D 2017-09-28T16:56:55.569
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 843ac757ad04480845ce4c2d2f742c13c8f751dec115a9a7d3dbef33c9264598
+F src/select.c f7bd3c62801cec412dbafde24585bc95300a909594b979764bd6ca0e61920f70
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 490e488ea963fe725b16212822c8608f2b6abce688931b611446bc2cbfe6b87c
-R 2f97a6d6fe698142318cfe94f9a0a57b
+P 5b9ae693120fe4f7bc3b6270f35d773876f6cc8f5990e05cce0d255c54b36ae7
+R d538249aab3ac2a14aedd6574010d06c
 U drh
-Z 00f2b9734ad71698420412023f6d2194
+Z 4f8ef3172b33aad0a660b2798399c834
diff --git a/manifest.uuid b/manifest.uuid
index 567c9cdf5d..e03321dd60 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5b9ae693120fe4f7bc3b6270f35d773876f6cc8f5990e05cce0d255c54b36ae7
\ No newline at end of file
+fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 79abda90a3..2c670ab6e3 100644
--- a/src/select.c
+++ b/src/select.c
@@ -118,7 +118,8 @@ Select *sqlite3SelectNew(
     pNew = &standin;
   }
   if( pEList==0 ){
-    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(pParse->db,TK_ASTERISK,0));
+    pEList = sqlite3ExprListAppend(pParse, 0,
+                                   sqlite3Expr(pParse->db,TK_ASTERISK,0));
   }
   pNew->pEList = pEList;
   pNew->op = TK_SELECT;
@@ -142,7 +143,8 @@ Select *sqlite3SelectNew(
   pNew->pLimit = pLimit;
   pNew->pOffset = pOffset;
   pNew->pWith = 0;
-  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || pParse->db->mallocFailed!=0 );
+  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0
+                     || pParse->db->mallocFailed!=0 );
   if( pParse->db->mallocFailed ) {
     clearSelect(pParse->db, pNew, pNew!=&standin);
     pNew = 0;
@@ -759,7 +761,8 @@ static void selectInnerLoop(
       assert( eDest==SRT_Set || eDest==SRT_Mem 
            || eDest==SRT_Coroutine || eDest==SRT_Output );
     }
-    nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,0,ecelFlags);
+    nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,
+                                         0,ecelFlags);
   }
 
   /* If the DISTINCT keyword was present on the SELECT statement
@@ -1570,9 +1573,9 @@ static void generateColumnTypes(
 **                              other words, the zSpan of the result expression.
 **
 **    short=ON, full=OFF:       (This is the default setting).  If the result
-**                              refers directly to a table column, then the result
-**                              column name is just the table column name: COLUMN. 
-**                              Otherwise use zSpan.
+**                              refers directly to a table column, then the
+**                              result column name is just the table column
+**                              name: COLUMN.  Otherwise use zSpan.
 **
 **    full=ON, short=ANY:       If the result refers directly to a table column,
 **                              then the result column name with the table name
@@ -1614,7 +1617,7 @@ static void generateColumnNames(
 
     assert( p!=0 );
     assert( p->op!=TK_AGG_COLUMN );  /* Agg processing has not run yet */
-    assert( p->op!=TK_COLUMN || p->pTab!=0 ); /* Covering indexes not yet coded */
+    assert( p->op!=TK_COLUMN || p->pTab!=0 ); /* Covering idx not yet coded */
     if( pEList->a[i].zName ){
       /* An AS clause always takes first priority */
       char *zName = pEList->a[i].zName;
@@ -3178,7 +3181,9 @@ static Expr *substExpr(
   Expr *pExpr            /* Expr in which substitution occurs */
 ){
   if( pExpr==0 ) return 0;
-  if( ExprHasProperty(pExpr, EP_FromJoin) && pExpr->iRightJoinTable==pSubst->iTable ){
+  if( ExprHasProperty(pExpr, EP_FromJoin)
+   && pExpr->iRightJoinTable==pSubst->iTable
+  ){
     pExpr->iRightJoinTable = pSubst->iNewTable;
   }
   if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){
@@ -3514,8 +3519,9 @@ static int flattenSubquery(
 #ifdef SQLITE_EXTRA_IFNULLROW
   else if( iFrom>0 && !isAgg ){
     /* Setting isLeftJoin to -1 causes OP_IfNullRow opcodes to be generated for
-    ** every reference to any result column from subquery in a join, even though
-    ** they are not necessary.  This will stress-test the OP_IfNullRow opcode. */
+    ** every reference to any result column from subquery in a join, even
+    ** though they are not necessary.  This will stress-test the OP_IfNullRow 
+    ** opcode. */
     isLeftJoin = -1;
   }
 #endif
@@ -4225,7 +4231,8 @@ static int withExpand(
       );
       return SQLITE_ERROR;
     }
-    assert( pTab->nTabRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 ));
+    assert( pTab->nTabRef==1 || 
+            ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 ));
 
     pCte->zCteErr = "circular reference: %s";
     pSavedWith = pParse->pWith;
@@ -5029,24 +5036,24 @@ static int countOfViewOptimization(Parse *pParse, Select *p){
   Expr *pExpr;
   Expr *pCount;
   sqlite3 *db;
-  if( (p->selFlags & SF_Aggregate)==0 ) return 0;   /* This is an aggregate query */
+  if( (p->selFlags & SF_Aggregate)==0 ) return 0;   /* This is an aggregate */
   if( p->pEList->nExpr!=1 ) return 0;               /* Single result column */
   pExpr = p->pEList->a[0].pExpr;
   if( pExpr->op!=TK_AGG_FUNCTION ) return 0;        /* Result is an aggregate */
-  if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0;  /* Must be count() */
+  if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0;  /* Is count() */
   if( pExpr->x.pList!=0 ) return 0;                 /* Must be count(*) */
-  if( p->pSrc->nSrc!=1 ) return 0;                  /* One table in the FROM clause */
+  if( p->pSrc->nSrc!=1 ) return 0;                  /* One table in FROM  */
   pSub = p->pSrc->a[0].pSelect;
   if( pSub==0 ) return 0;                           /* The FROM is a subquery */
-  if( pSub->pPrior==0 ) return 0;                   /* Must be a compound subquery */
+  if( pSub->pPrior==0 ) return 0;                   /* Must be a compound ry */
   do{
     if( pSub->op!=TK_ALL && pSub->pPrior ) return 0;  /* Must be UNION ALL */
     if( pSub->pWhere ) return 0;                      /* No WHERE clause */
     if( pSub->selFlags & SF_Aggregate ) return 0;     /* Not an aggregate */
-    pSub = pSub->pPrior;                              /* Repeat over compound terms */
+    pSub = pSub->pPrior;                              /* Repeat over compound */
   }while( pSub );
 
-  /* If we reach this point, that means it is OK to perform the transformation */
+  /* If we reach this point then it is OK to perform the transformation */
 
   db = pParse->db;
   pCount = pExpr;
@@ -5239,9 +5246,10 @@ int sqlite3Select(
     SelectDest dest;
     Select *pSub;
 
-    /* Issue SQLITE_READ authorizations with a fake column name for any tables that
-    ** are referenced but from which no values are extracted. Examples of where these
-    ** kinds of null SQLITE_READ authorizations would occur:
+    /* Issue SQLITE_READ authorizations with a fake column name for any
+    ** tables that are referenced but from which no values are extracted.
+    ** Examples of where these kinds of null SQLITE_READ authorizations
+    ** would occur:
     **
     **     SELECT count(*) FROM t1;   -- SQLITE_READ t1.""
     **     SELECT t1.* FROM t1, t2;   -- SQLITE_READ t2.""
@@ -5249,10 +5257,10 @@ int sqlite3Select(
     ** The fake column name is an empty string.  It is possible for a table to
     ** have a column named by the empty string, in which case there is no way to
     ** distinguish between an unreferenced table and an actual reference to the
-    ** "" column.  The original design was for the fake column name to be a NULL,
+    ** "" column. The original design was for the fake column name to be a NULL,
     ** which would be unambiguous.  But legacy authorization callbacks might
-    ** assume the column name is non-NULL and segfault.  The use of an empty string
-    ** for the fake column name seems safer.
+    ** assume the column name is non-NULL and segfault.  The use of an empty
+    ** string for the fake column name seems safer.
     */
     if( pItem->colUsed==0 ){
       sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase);

From 2e9d706252dd774399fe8a015aebf540d820d10c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 17:29:24 +0000
Subject: [PATCH 125/270] Remove the (undocumented) query-planner control that
 prevents a "SELECT ALL" subquery in FROM clause from being implemented as a
 co-routine.

FossilOrigin-Name: ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  6 ++----
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index f258da1133..d11bdc4fb7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sover-length\ssource\scode\slines\sin\sselect.c.\s\sNo\slogic\schanges.
-D 2017-09-28T16:56:55.569
+C Remove\sthe\s(undocumented)\squery-planner\scontrol\sthat\sprevents\s\na\s"SELECT\sALL"\ssubquery\sin\sFROM\sclause\sfrom\sbeing\simplemented\sas\sa\sco-routine.
+D 2017-09-28T17:29:24.426
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c f7bd3c62801cec412dbafde24585bc95300a909594b979764bd6ca0e61920f70
+F src/select.c ff9551b1792d838d1973ca3f5062804e2a00b1fa2cef6d7931992d89f2b3744a
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5b9ae693120fe4f7bc3b6270f35d773876f6cc8f5990e05cce0d255c54b36ae7
-R d538249aab3ac2a14aedd6574010d06c
+P fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
+R 5a9f08df25678ade0635d27ff9994d5e
 U drh
-Z 4f8ef3172b33aad0a660b2798399c834
+Z 15e07447876306f708cc7a309f77f755
diff --git a/manifest.uuid b/manifest.uuid
index e03321dd60..6515252213 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
\ No newline at end of file
+ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 2c670ab6e3..a5f04767bf 100644
--- a/src/select.c
+++ b/src/select.c
@@ -5317,9 +5317,7 @@ int sqlite3Select(
     ** The subquery is implemented as a co-routine if all of these are true:
     **   (1)  The subquery is guaranteed to be the outer loop (so that it
     **        does not need to be computed more than once)
-    **   (2)  The ALL keyword after SELECT is omitted.  (Applications are
-    **        allowed to say "SELECT ALL" instead of just "SELECT" to disable
-    **        the use of co-routines.)
+    **   (2)  REMOVED (2017-09-28): The ALL keyword after SELECT is omitted.
     **   (3)  Co-routines are not disabled using sqlite3_test_control()
     **        with SQLITE_TESTCTRL_OPTIMIZATIONS.
     **
@@ -5329,7 +5327,7 @@ int sqlite3Select(
     if( i==0
      && (pTabList->nSrc==1
             || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0)  /* (1) */
-     && (p->selFlags & SF_All)==0                                   /* (2) */
+     /*** constraint removed: && (p->selFlags & SF_All)==0             (2) */
      && OptimizationEnabled(db, SQLITE_SubqCoroutine)               /* (3) */
     ){
       /* Implement a co-routine that will return a single row of the result

From 648fe49f96dc485e9ab0d2a7b6c4c0e69cdaac85 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 28 Sep 2017 20:06:53 +0000
Subject: [PATCH 126/270] Do not flatten subqueries that contain an ORDER BY or
 GROUP BY clause and can be implemented using a co-routine.

FossilOrigin-Name: 042d655dd9002e8b89a798ad955b0285891aecf79f6978c5312e70ffe0609a46
---
 manifest      | 15 +++++++++------
 manifest.uuid |  2 +-
 src/select.c  | 19 +++++++++++++++++++
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index d11bdc4fb7..c63192e1bf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\s(undocumented)\squery-planner\scontrol\sthat\sprevents\s\na\s"SELECT\sALL"\ssubquery\sin\sFROM\sclause\sfrom\sbeing\simplemented\sas\sa\sco-routine.
-D 2017-09-28T17:29:24.426
+C Do\snot\sflatten\ssubqueries\sthat\scontain\san\sORDER\sBY\sor\sGROUP\sBY\sclause\sand\ncan\sbe\simplemented\susing\sa\sco-routine.
+D 2017-09-28T20:06:53.014
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c ff9551b1792d838d1973ca3f5062804e2a00b1fa2cef6d7931992d89f2b3744a
+F src/select.c d38163a76a0da425afd4a43f5dd0dc88fdbe0aa258c45f6a9cb44acf0a606bc5
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
-R 5a9f08df25678ade0635d27ff9994d5e
+P ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
+R 2327508642786ef6d3dd975eaebc22c4
+T *branch * prefer-coroutine-sort-subquery
+T *sym-prefer-coroutine-sort-subquery *
+T -sym-trunk *
 U drh
-Z 15e07447876306f708cc7a309f77f755
+Z 34c1093fc7e9ecc2e1ce3f4d0b9d734a
diff --git a/manifest.uuid b/manifest.uuid
index 6515252213..61c790dd19 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
\ No newline at end of file
+042d655dd9002e8b89a798ad955b0285891aecf79f6978c5312e70ffe0609a46
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index a5f04767bf..7583fc24f1 100644
--- a/src/select.c
+++ b/src/select.c
@@ -5205,6 +5205,25 @@ int sqlite3Select(
       goto select_end;
     }
 
+    /* If the subquery contains an ORDER BY or GROUP BY clause and if
+    ** it will be implemented as a co-routine, then do not flatten.  This
+    ** restriction allows SQL constructs like this:
+    **
+    **  SELECT expensive_function(x)
+    **    FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
+    **
+    ** The expensive_function() is only computed on the 10 rows that
+    ** are output, rather than every row of the table.
+    */
+    if( (pSub->pOrderBy!=0 || pSub->pGroupBy!=0)
+     && i==0
+     && (pTabList->nSrc==1
+         || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0)
+     && OptimizationEnabled(db, SQLITE_SubqCoroutine)
+    ){
+      continue;
+    }
+
     isAggSub = (pSub->selFlags & SF_Aggregate)!=0;
     if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){
       /* This subquery can be absorbed into its parent. */

From 23768298d222719eb135b7320170a303b3248b03 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Sep 2017 12:12:52 +0000
Subject: [PATCH 127/270] Better names for subqueries in EXPLAIN comments.

FossilOrigin-Name: 04ef40a8fea88776e0d8b73d942922d45d0c038fbfac1a9a82b9322181b4ad92
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 src/select.c      |  6 +++++-
 test/select1.test |  4 ++--
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index d11bdc4fb7..98cc331fcd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\s(undocumented)\squery-planner\scontrol\sthat\sprevents\s\na\s"SELECT\sALL"\ssubquery\sin\sFROM\sclause\sfrom\sbeing\simplemented\sas\sa\sco-routine.
-D 2017-09-28T17:29:24.426
+C Better\snames\sfor\ssubqueries\sin\sEXPLAIN\scomments.
+D 2017-09-29T12:12:52.366
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c ff9551b1792d838d1973ca3f5062804e2a00b1fa2cef6d7931992d89f2b3744a
+F src/select.c 1508104113d8d1d1eef0b69255abc9816ce5b02e73a4962f29b85112723d8be8
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1158,7 +1158,7 @@ F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
 F test/schema6.test 5b21bbdd405bc93b3e6af5e6ece64d230e35f65cc4035e5c2b89fc8a090d7270
 F test/securedel.test 5f997cb6bd38727b81e0985f53ec386c99db6441b2b9e6357240649d29017239
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
-F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
+F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7
 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56
 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054
 F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf
-R 5a9f08df25678ade0635d27ff9994d5e
+P ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
+R 24b1805bda8bda8fe8857c0377afb6ae
 U drh
-Z 15e07447876306f708cc7a309f77f755
+Z a390b9b3df04d20b4fc367656c38993e
diff --git a/manifest.uuid b/manifest.uuid
index 6515252213..450d0bf34e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
\ No newline at end of file
+04ef40a8fea88776e0d8b73d942922d45d0c038fbfac1a9a82b9322181b4ad92
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index a5f04767bf..9c671bf35b 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4376,7 +4376,11 @@ static int selectExpander(Walker *pWalker, Select *p){
       pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
       if( pTab==0 ) return WRC_Abort;
       pTab->nTabRef = 1;
-      pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab);
+      if( pFrom->zAlias ){
+        pTab->zName = sqlite3DbStrDup(db, pFrom->zAlias);
+      }else{
+        pTab->zName = sqlite3MPrintf(db, "subquery_%p", (void*)pTab);
+      }
       while( pSel->pPrior ){ pSel = pSel->pPrior; }
       sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol);
       pTab->iPKey = -1;
diff --git a/test/select1.test b/test/select1.test
index 4d6c07f2d0..43b20f6d15 100644
--- a/test/select1.test
+++ b/test/select1.test
@@ -545,9 +545,9 @@ do_test select1-6.9.7 {
   set x [execsql2 {
      SELECT * FROM test1 a, (select 5, 6) LIMIT 1
   }]
-  regsub -all {sq_[0-9a-fA-F_]+} $x {subquery} x
+  regsub -all {subquery_[0-9a-fA-F_]+} $x {subquery} x
   set x
-} {a.f1 11 a.f2 22 sqlite_subquery.5 5 sqlite_subquery.6 6}
+} {a.f1 11 a.f2 22 subquery.5 5 subquery.6 6}
 do_test select1-6.9.8 {
   set x [execsql2 {
      SELECT * FROM test1 a, (select 5 AS x, 6 AS y) AS b LIMIT 1

From 824d21aff3c6f1f38437e22744defc09d1f8085c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Sep 2017 12:44:52 +0000
Subject: [PATCH 128/270] Make sure the 6th parameter to the authorizer
 callback for view subqueries has the correct view name.

FossilOrigin-Name: 2a45bbc9fd1c64f1c4c4dac38f063cd67480fcb6da24bf93fdefbfca66fd81ab
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  8 ++++++++
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 98cc331fcd..f377f6008f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Better\snames\sfor\ssubqueries\sin\sEXPLAIN\scomments.
-D 2017-09-29T12:12:52.366
+C Make\ssure\sthe\s6th\sparameter\sto\sthe\sauthorizer\scallback\sfor\sview\ssubqueries\nhas\sthe\scorrect\sview\sname.
+D 2017-09-29T12:44:52.580
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 1508104113d8d1d1eef0b69255abc9816ce5b02e73a4962f29b85112723d8be8
+F src/select.c 61f6ee59115ddba550504296587571fc82acc2a8f178a5cd172c7fff13f83685
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18
-R 24b1805bda8bda8fe8857c0377afb6ae
+P 04ef40a8fea88776e0d8b73d942922d45d0c038fbfac1a9a82b9322181b4ad92
+R 366bed71d0ebda5b55c786e6b191d9a3
 U drh
-Z a390b9b3df04d20b4fc367656c38993e
+Z 1e56973e5f1cf4ced8af06a37f0fc40f
diff --git a/manifest.uuid b/manifest.uuid
index 450d0bf34e..448bac01f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04ef40a8fea88776e0d8b73d942922d45d0c038fbfac1a9a82b9322181b4ad92
\ No newline at end of file
+2a45bbc9fd1c64f1c4c4dac38f063cd67480fcb6da24bf93fdefbfca66fd81ab
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 9c671bf35b..ee949c881d 100644
--- a/src/select.c
+++ b/src/select.c
@@ -5249,6 +5249,9 @@ int sqlite3Select(
     struct SrcList_item *pItem = &pTabList->a[i];
     SelectDest dest;
     Select *pSub;
+#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
+    const char *zSavedAuthContext;
+#endif
 
     /* Issue SQLITE_READ authorizations with a fake column name for any
     ** tables that are referenced but from which no values are extracted.
@@ -5316,6 +5319,9 @@ int sqlite3Select(
 #endif
     }
 
+    zSavedAuthContext = pParse->zAuthContext;
+    pParse->zAuthContext = pItem->zName;
+
     /* Generate code to implement the subquery
     **
     ** The subquery is implemented as a co-routine if all of these are true:
@@ -5338,6 +5344,7 @@ int sqlite3Select(
       ** set on each invocation.
       */
       int addrTop = sqlite3VdbeCurrentAddr(v)+1;
+     
       pItem->regReturn = ++pParse->nMem;
       sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop);
       VdbeComment((v, "%s", pItem->pTab->zName));
@@ -5395,6 +5402,7 @@ int sqlite3Select(
     }
     if( db->mallocFailed ) goto select_end;
     pParse->nHeight -= sqlite3SelectExprHeight(p);
+    pParse->zAuthContext = zSavedAuthContext;
 #endif
   }
 

From d981e8289afed9ab16d111016c329463a65dfa2f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Sep 2017 16:07:56 +0000
Subject: [PATCH 129/270] Clean up the comments on the query flattener to more
 clearly and accurately express the conditions under which the flattener is
 able to run.

FossilOrigin-Name: 0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86
---
 manifest      |  12 +++---
 manifest.uuid |   2 +-
 src/select.c  | 113 ++++++++++++++++++++++++++------------------------
 3 files changed, 66 insertions(+), 61 deletions(-)

diff --git a/manifest b/manifest
index f377f6008f..87b5990a98 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\s6th\sparameter\sto\sthe\sauthorizer\scallback\sfor\sview\ssubqueries\nhas\sthe\scorrect\sview\sname.
-D 2017-09-29T12:44:52.580
+C Clean\sup\sthe\scomments\son\sthe\squery\sflattener\sto\smore\sclearly\sand\saccurately\nexpress\sthe\sconditions\sunder\swhich\sthe\sflattener\sis\sable\sto\srun.
+D 2017-09-29T16:07:56.167
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 61f6ee59115ddba550504296587571fc82acc2a8f178a5cd172c7fff13f83685
+F src/select.c 733e7f727d4f83234983fcbcb71670229f6d9ae78ee724fbf40b4aa6904474e2
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04ef40a8fea88776e0d8b73d942922d45d0c038fbfac1a9a82b9322181b4ad92
-R 366bed71d0ebda5b55c786e6b191d9a3
+P 2a45bbc9fd1c64f1c4c4dac38f063cd67480fcb6da24bf93fdefbfca66fd81ab
+R 5dd9d1abad7c6b3045de562026df2e52
 U drh
-Z 1e56973e5f1cf4ced8af06a37f0fc40f
+Z 40508487cfccebe9fe70dcdbc46f56d0
diff --git a/manifest.uuid b/manifest.uuid
index 448bac01f9..48e1728350 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2a45bbc9fd1c64f1c4c4dac38f063cd67480fcb6da24bf93fdefbfca66fd81ab
\ No newline at end of file
+0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index ee949c881d..edf55a5c7b 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3296,68 +3296,71 @@ static void substSelect(
 ** exist on the table t1, a complete scan of the data might be
 ** avoided.
 **
-** Flattening is only attempted if all of the following are true:
+** Flattening is subject to the following constraints:
 **
-**   (1)  The subquery and the outer query do not both use aggregates.
+**   (1)  The subquery and the outer query cannot both be aggregates.
 **
-**   (2)  The subquery is not an aggregate or (2a) the outer query is not a join
-**        and (2b) the outer query does not use subqueries other than the one
-**        FROM-clause subquery that is a candidate for flattening.  (2b is
-**        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
+**   (2)  If the subquery is an aggregate then
+**        (2a) the outer query must not be a join and
+**        (2b) the outer query must not use subqueries
+**             other than the one FROM-clause subquery that is a candidate
+**             for flattening.  (This is due to ticket [2f7170d73bf9abf80]
+**             from 2015-02-09.)
 **
-**   (3)  The subquery is not the right operand of a LEFT JOIN
-**        or (a) the subquery is not itself a join and (b) the FROM clause
-**        of the subquery does not contain a virtual table and (c) the 
-**        outer query is not an aggregate.
+**   (3)  If the subquery is the right operand of a LEFT JOIN then
+**        (3a) the subquery may not be a join and
+**        (3b) the FROM clause of the subquery may not contain a virtual
+**             table and
+**        (3c) the outer query may not be an aggregate.
 **
-**   (4)  The subquery is not DISTINCT.
+**   (4)  The subquery can not be DISTINCT.
 **
 **  (**)  At one point restrictions (4) and (5) defined a subset of DISTINCT
 **        sub-queries that were excluded from this optimization. Restriction 
 **        (4) has since been expanded to exclude all DISTINCT subqueries.
 **
-**   (6)  The subquery does not use aggregates or the outer query is not
-**        DISTINCT.
+**   (6)  If the subquery is aggregate, the outer query may not be DISTINCT.
 **
-**   (7)  The subquery has a FROM clause.  TODO:  For subqueries without
+**   (7)  The subquery must have a FROM clause.  TODO:  For subqueries without
 **        A FROM clause, consider adding a FROM clause with the special
 **        table sqlite_once that consists of a single row containing a
 **        single NULL.
 **
-**   (8)  The subquery does not use LIMIT or the outer query is not a join.
+**   (8)  If the subquery uses LIMIT then the outer query may not be a join.
 **
-**   (9)  The subquery does not use LIMIT or the outer query does not use
-**        aggregates.
+**   (9)  If the subquery uses LIMIT then the outer query may not be aggregate.
 **
 **  (**)  Restriction (10) was removed from the code on 2005-02-05 but we
 **        accidently carried the comment forward until 2014-09-15.  Original
-**        text: "The subquery does not use aggregates or the outer query 
-**        does not use LIMIT."
+**        constraint: "If the subquery is aggregate then the outer query 
+**        may not use LIMIT."
 **
-**  (11)  The subquery and the outer query do not both have ORDER BY clauses.
+**  (11)  The subquery and the outer query may not both have ORDER BY clauses.
 **
 **  (**)  Not implemented.  Subsumed into restriction (3).  Was previously
 **        a separate restriction deriving from ticket #350.
 **
-**  (13)  The subquery and outer query do not both use LIMIT.
+**  (13)  The subquery and outer query may not both use LIMIT.
 **
-**  (14)  The subquery does not use OFFSET.
+**  (14)  The subquery may not use OFFSET.
 **
-**  (15)  The outer query is not part of a compound select or the
-**        subquery does not have a LIMIT clause.
+**  (15)  If the outer query is part of a compound select, then the
+**        subquery may not use LIMIT.
 **        (See ticket #2339 and ticket [02a8e81d44]).
 **
-**  (16)  The outer query is not an aggregate or the subquery does
-**        not contain ORDER BY.  (Ticket #2942)  This used to not matter
+**  (16)  If the outer query is aggregate, then the subquery may not
+**        use ORDER BY.  (Ticket #2942)  This used to not matter
 **        until we introduced the group_concat() function.  
 **
-**  (17)  The sub-query is not a compound select, or it is a UNION ALL 
-**        compound clause made up entirely of non-aggregate queries, and 
-**        the parent query:
-**
-**          * is not itself part of a compound select,
-**          * is not an aggregate or DISTINCT query, and
-**          * is not a join
+**  (17)  If the subquery is a compound select, then
+**        (17a) all compound operators must be a UNION ALL, and
+**        (17b) no terms within the subquery compound may be aggregate
+**              or DISTINT, and
+**        (17c) every term within the subquery compound must have a FROM clause
+**        (17d) the outer query may not be
+**              (17d1) aggregate, or
+**              (17d2) DISTINCT, or
+**              (17d3) a join.
 **
 **        The parent and sub-query may contain WHERE clauses. Subject to
 **        rules (11), (13) and (14), they may also contain ORDER BY,
@@ -3373,10 +3376,10 @@ static void substSelect(
 **        syntax error and return a detailed message.
 **
 **  (18)  If the sub-query is a compound select, then all terms of the
-**        ORDER by clause of the parent must be simple references to 
+**        ORDER BY clause of the parent must be simple references to 
 **        columns of the sub-query.
 **
-**  (19)  The subquery does not use LIMIT or the outer query does not
+**  (19)  If the subquery uses LIMIT then the outer query may not
 **        have a WHERE clause.
 **
 **  (20)  If the sub-query is a compound select, then it must not use
@@ -3385,17 +3388,17 @@ static void substSelect(
 **        appear as unmodified result columns in the outer query.  But we
 **        have other optimizations in mind to deal with that case.
 **
-**  (21)  The subquery does not use LIMIT or the outer query is not
+**  (21)  If the subquery uses LIMIT then the outer query may not be
 **        DISTINCT.  (See ticket [752e1646fc]).
 **
-**  (22)  The subquery is not a recursive CTE.
+**  (22)  The subquery may not be a recursive CTE.
 **
-**  (23)  The parent is not a recursive CTE, or the sub-query is not a
-**        compound query. This restriction is because transforming the
+**  (23)  If the outer query is a recursive CTE, then the sub-query may not be
+**        a compound query. This restriction is because transforming the
 **        parent to a compound query confuses the code that handles
 **        recursive queries in multiSelect().
 **
-**  (24)  The subquery is not an aggregate that uses the built-in min() or 
+**  (24)  The subquery may not be an aggregate that uses the built-in min() or 
 **        or max() functions.  (Without this restriction, a query like:
 **        "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily
 **        return the value X for which Y was maximal.)
@@ -3435,7 +3438,7 @@ static int flattenSubquery(
   /* Check to see if flattening is permitted.  Return 0 if not.
   */
   assert( p!=0 );
-  assert( p->pPrior==0 );  /* Unable to flatten compound queries */
+  assert( p->pPrior==0 );
   if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0;
   pSrc = p->pSrc;
   assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc );
@@ -3467,7 +3470,7 @@ static int flattenSubquery(
     return 0;                                            /* Restriction (15) */
   }
   if( pSubSrc->nSrc==0 ) return 0;                       /* Restriction (7)  */
-  if( pSub->selFlags & SF_Distinct ) return 0;           /* Restriction (5)  */
+  if( pSub->selFlags & SF_Distinct ) return 0;           /* Restriction (4)  */
   if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){
      return 0;         /* Restrictions (8)(9) */
   }
@@ -3493,7 +3496,8 @@ static int flattenSubquery(
 
   /*
   ** If the subquery is the right operand of a LEFT JOIN, then the
-  ** subquery may not be a join itself.  Example of why this is not allowed:
+  ** subquery may not be a join itself (3a). Example of why this is not
+  ** allowed:
   **
   **         t1 LEFT OUTER JOIN (t2 JOIN t3)
   **
@@ -3504,16 +3508,17 @@ static int flattenSubquery(
   ** which is not at all the same thing.
   **
   ** If the subquery is the right operand of a LEFT JOIN, then the outer
-  ** query cannot be an aggregate.  This is an artifact of the way aggregates
-  ** are processed - there is no mechanism to determine if the LEFT JOIN
-  ** table should be all-NULL.
+  ** query cannot be an aggregate. (3c)  This is an artifact of the way
+  ** aggregates are processed - there is no mechanism to determine if
+  ** the LEFT JOIN table should be all-NULL.
   **
   ** See also tickets #306, #350, and #3300.
   */
   if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){
     isLeftJoin = 1;
     if( pSubSrc->nSrc>1 || isAgg || IsVirtual(pSubSrc->a[0].pTab) ){
-      return 0; /* Restriction (3) */
+      /*  (3a)             (3c)     (3b) */
+      return 0;
     }
   }
 #ifdef SQLITE_EXTRA_IFNULLROW
@@ -3526,33 +3531,33 @@ static int flattenSubquery(
   }
 #endif
 
-  /* Restriction 17: If the sub-query is a compound SELECT, then it must
+  /* Restriction (17): If the sub-query is a compound SELECT, then it must
   ** use only the UNION ALL operator. And none of the simple select queries
   ** that make up the compound SELECT are allowed to be aggregate or distinct
   ** queries.
   */
   if( pSub->pPrior ){
     if( pSub->pOrderBy ){
-      return 0;  /* Restriction 20 */
+      return 0;  /* Restriction (20) */
     }
     if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
-      return 0;
+      return 0; /* (17d1), (17d2), or (17d3) */
     }
     for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){
       testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct );
       testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate );
       assert( pSub->pSrc!=0 );
       assert( pSub->pEList->nExpr==pSub1->pEList->nExpr );
-      if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0
-       || (pSub1->pPrior && pSub1->op!=TK_ALL) 
-       || pSub1->pSrc->nSrc<1
+      if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0    /* (17b) */
+       || (pSub1->pPrior && pSub1->op!=TK_ALL)                 /* (17a) */
+       || pSub1->pSrc->nSrc<1                                  /* (17c) */
       ){
         return 0;
       }
       testcase( pSub1->pSrc->nSrc>1 );
     }
 
-    /* Restriction 18. */
+    /* Restriction (18). */
     if( p->pOrderBy ){
       int ii;
       for(ii=0; ii<p->pOrderBy->nExpr; ii++){

From 25c221eb1dcc72ac0e524c605d61042ae91a6fe8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Sep 2017 22:13:24 +0000
Subject: [PATCH 130/270] Always render a subquery that is not part of a join
 as a co-routine.

FossilOrigin-Name: 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37
---
 manifest        | 16 ++++-----
 manifest.uuid   |  2 +-
 src/select.c    | 94 +++++++++++++++++++------------------------------
 src/sqliteInt.h | 18 +++++-----
 src/test1.c     |  1 -
 5 files changed, 53 insertions(+), 78 deletions(-)

diff --git a/manifest b/manifest
index 26986bd8f3..35a58bfa19 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\squery\sflattener\scomment\simprovements\sfrom\strunk.
-D 2017-09-29T16:08:46.956
+C Always\srender\sa\ssubquery\sthat\sis\snot\spart\sof\sa\sjoin\sas\sa\sco-routine.
+D 2017-09-29T22:13:24.684
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,18 +458,18 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 73fa1cc03a325dd5f3e84bc6c4c975000028f96f85025ae2fe0b6eb0c30c2fe0
+F src/select.c 8e8e9965af11f7ca54a44fc2cea9e8d162a6ffa1ccd2e840c69ff4468e0d08e0
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 954fed875c59d283870f13010bbbb96d8b25cf3db0683843403c36e9dfcf1bdd
+F src/sqliteInt.h d51b1b549b6f64a44b0f5f5620aeb5bba414cf07704978238885b273ce2aa84f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828
-F src/test1.c a947b2554fa77d0ef2dd21d1ef08e37e5d91b17af83de923a4e3c7f10957a2eb
+F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 06f432fb7c979f1bb7f01f5c90716ce5c0248f73f70b78a9870b9de5c9bf7ef4 0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86
-R 4be3453c15fd85db566735e226eb00f9
+P f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b
+R 5c67a1a4877d581279b4ed945a250449
 U drh
-Z be149bf809ee0d7792ea4e04a36ef7f0
+Z 6d48696ae0bb75b33e85df88734665c6
diff --git a/manifest.uuid b/manifest.uuid
index bef919b1e8..143260d747 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b
\ No newline at end of file
+6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index adc23d887a..0f46d73f31 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3298,9 +3298,11 @@ static void substSelect(
 **
 ** Flattening is subject to the following constraints:
 **
-**   (1)  The subquery and the outer query cannot both be aggregates.
+**  (**)  We no longer attempt to flatten aggregate subqueries. Was:
+**        The subquery and the outer query cannot both be aggregates.
 **
-**   (2)  If the subquery is an aggregate then
+**  (**)  We no longer attempt to flatten aggregate subqueries. Was:
+**        (2) If the subquery is an aggregate then
 **        (2a) the outer query must not be a join and
 **        (2b) the outer query must not use subqueries
 **             other than the one FROM-clause subquery that is a candidate
@@ -3319,7 +3321,8 @@ static void substSelect(
 **        sub-queries that were excluded from this optimization. Restriction 
 **        (4) has since been expanded to exclude all DISTINCT subqueries.
 **
-**   (6)  If the subquery is aggregate, the outer query may not be DISTINCT.
+**  (**)  We no longer attempt to flatten aggregate subqueries.  Was:
+**        If the subquery is aggregate, the outer query may not be DISTINCT.
 **
 **   (7)  The subquery must have a FROM clause.  TODO:  For subqueries without
 **        A FROM clause, consider adding a FROM clause with the special
@@ -3406,7 +3409,7 @@ static void substSelect(
 **
 ** In this routine, the "p" parameter is a pointer to the outer query.
 ** The subquery is p->pSrc->a[iFrom].  isAgg is true if the outer query
-** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates.
+** uses aggregates.
 **
 ** If flattening is not attempted, this routine is a no-op and returns 0.
 ** If flattening is attempted this routine returns 1.
@@ -3418,8 +3421,7 @@ static int flattenSubquery(
   Parse *pParse,       /* Parsing context */
   Select *p,           /* The parent or outer SELECT statement */
   int iFrom,           /* Index in p->pSrc->a[] of the inner subquery */
-  int isAgg,           /* True if outer SELECT uses aggregate functions */
-  int subqueryIsAgg    /* True if the subquery uses aggregate functions */
+  int isAgg            /* True if outer SELECT uses aggregate functions */
 ){
   const char *zSavedAuthContext = pParse->zAuthContext;
   Select *pParent;    /* Current UNION ALL term of the other query */
@@ -3446,16 +3448,6 @@ static int flattenSubquery(
   iParent = pSubitem->iCursor;
   pSub = pSubitem->pSelect;
   assert( pSub!=0 );
-  if( subqueryIsAgg ){
-    if( isAgg ) return 0;                                /* Restriction (1)   */
-    if( pSrc->nSrc>1 ) return 0;                         /* Restriction (2a)  */
-    if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery))
-     || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0
-     || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0
-    ){
-      return 0;                                          /* Restriction (2b)  */
-    }
-  }
 
   pSubSrc = pSub->pSrc;
   assert( pSubSrc );
@@ -3474,9 +3466,6 @@ static int flattenSubquery(
   if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){
      return 0;         /* Restrictions (8)(9) */
   }
-  if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){
-     return 0;         /* Restriction (6)  */
-  }
   if( p->pOrderBy && pSub->pOrderBy ){
      return 0;                                           /* Restriction (11) */
   }
@@ -3778,18 +3767,7 @@ static int flattenSubquery(
     if( isLeftJoin>0 ){
       setJoinExpr(pWhere, iNewParent);
     }
-    if( subqueryIsAgg ){
-      assert( pParent->pHaving==0 );
-      pParent->pHaving = pParent->pWhere;
-      pParent->pWhere = pWhere;
-      pParent->pHaving = sqlite3ExprAnd(db, 
-          sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving
-      );
-      assert( pParent->pGroupBy==0 );
-      pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0);
-    }else{
-      pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
-    }
+    pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
     if( db->mallocFailed==0 ){
       SubstContext x;
       x.pParse = pParse;
@@ -3852,9 +3830,9 @@ static int flattenSubquery(
 **
 ** Do not attempt this optimization if:
 **
-**   (1) The inner query is an aggregate.  (In that case, we'd really want
-**       to copy the outer WHERE-clause terms onto the HAVING clause of the
-**       inner query.  But they probably won't help there so do not bother.)
+**   (1) (** This restriction was removed on 2017-09-29.  We used to
+**           disallow this optimization for aggregate subqueries, but now
+**           it is allowed by putting the extra terms on the HAVING clause **)
 **
 **   (2) The inner query is the recursive part of a common table expression.
 **
@@ -3882,11 +3860,9 @@ static int pushDownWhereTerms(
   Select *pX;           /* For looping over compound SELECTs in pSubq */
   if( pWhere==0 ) return 0;
   for(pX=pSubq; pX; pX=pX->pPrior){
-    if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){
-      testcase( pX->selFlags & SF_Aggregate );
-      testcase( pX->selFlags & SF_Recursive );
+    if( (pX->selFlags & (SF_Recursive))!=0 ){
       testcase( pX!=pSubq );
-      return 0; /* restrictions (1) and (2) */
+      return 0; /* restriction (2) */
     }
   }
   if( pSubq->pLimit!=0 ){
@@ -3896,7 +3872,7 @@ static int pushDownWhereTerms(
     nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor);
     pWhere = pWhere->pLeft;
   }
-  if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */
+  if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction (5) */
   if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){
     nChng++;
     while( pSubq ){
@@ -3908,7 +3884,11 @@ static int pushDownWhereTerms(
       x.isLeftJoin = 0;
       x.pEList = pSubq->pEList;
       pNew = substExpr(&x, pNew);
-      pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
+      if( pSubq->selFlags & SF_Aggregate ){
+        pSubq->pHaving = sqlite3ExprAnd(pParse->db, pSubq->pHaving, pNew);
+      }else{
+        pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
+      }
       pSubq = pSubq->pPrior;
     }
   }
@@ -5202,7 +5182,6 @@ int sqlite3Select(
   for(i=0; !p->pPrior && i<pTabList->nSrc; i++){
     struct SrcList_item *pItem = &pTabList->a[i];
     Select *pSub = pItem->pSelect;
-    int isAggSub;
     Table *pTab = pItem->pTab;
     if( pSub==0 ) continue;
 
@@ -5214,7 +5193,17 @@ int sqlite3Select(
       goto select_end;
     }
 
-    /* If the subquery contains an ORDER BY or GROUP BY clause and if
+    /* Do not try to flatten an aggregate subquery.
+    **
+    ** Flattening an aggregate subquery is only possible if the outer query
+    ** is not a join.  But if the outer query is not a join, then the subquery
+    ** will be implemented as a co-routine and there is no advantage to
+    ** flattening in that case.
+    */
+    if( (pSub->selFlags & SF_Aggregate)!=0 ) continue;
+    assert( pSub->pGroupBy==0 );
+
+    /* If the subquery contains an ORDER BY clause and if
     ** it will be implemented as a co-routine, then do not flatten.  This
     ** restriction allows SQL constructs like this:
     **
@@ -5224,22 +5213,16 @@ int sqlite3Select(
     ** The expensive_function() is only computed on the 10 rows that
     ** are output, rather than every row of the table.
     */
-    if( (pSub->pOrderBy!=0 || pSub->pGroupBy!=0)
+    if( pSub->pOrderBy!=0
      && i==0
      && (pTabList->nSrc==1
          || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0)
-     && OptimizationEnabled(db, SQLITE_SubqCoroutine)
     ){
       continue;
     }
 
-    isAggSub = (pSub->selFlags & SF_Aggregate)!=0;
-    if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){
+    if( flattenSubquery(pParse, p, i, isAgg) ){
       /* This subquery can be absorbed into its parent. */
-      if( isAggSub ){
-        isAgg = 1;
-        p->selFlags |= SF_Aggregate;
-      }
       i = -1;
     }
     pTabList = p->pSrc;
@@ -5348,12 +5331,9 @@ int sqlite3Select(
 
     /* Generate code to implement the subquery
     **
-    ** The subquery is implemented as a co-routine if all of these are true:
-    **   (1)  The subquery is guaranteed to be the outer loop (so that it
-    **        does not need to be computed more than once)
-    **   (2)  REMOVED (2017-09-28): The ALL keyword after SELECT is omitted.
-    **   (3)  Co-routines are not disabled using sqlite3_test_control()
-    **        with SQLITE_TESTCTRL_OPTIMIZATIONS.
+    ** The subquery is implemented as a co-routine if the subquery is
+    ** guaranteed to be the outer loop (so that it does not need to be
+    ** computed more than once)
     **
     ** TODO: Are there other reasons beside (1) to use a co-routine
     ** implementation?
@@ -5361,8 +5341,6 @@ int sqlite3Select(
     if( i==0
      && (pTabList->nSrc==1
             || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0)  /* (1) */
-     /*** constraint removed: && (p->selFlags & SF_All)==0             (2) */
-     && OptimizationEnabled(db, SQLITE_SubqCoroutine)               /* (3) */
     ){
       /* Implement a co-routine that will return a single row of the result
       ** set on each invocation.
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 3259df894f..9b45295e70 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1506,16 +1506,14 @@ struct sqlite3 {
 #define SQLITE_ColumnCache    0x0002   /* Column cache */
 #define SQLITE_GroupByOrder   0x0004   /* GROUPBY cover of ORDERBY */
 #define SQLITE_FactorOutConst 0x0008   /* Constant factoring */
-/*                not used    0x0010   // Was: SQLITE_IdxRealAsInt */
-#define SQLITE_DistinctOpt    0x0020   /* DISTINCT using indexes */
-#define SQLITE_CoverIdxScan   0x0040   /* Covering index scans */
-#define SQLITE_OrderByIdxJoin 0x0080   /* ORDER BY of joins via index */
-#define SQLITE_SubqCoroutine  0x0100   /* Evaluate subqueries as coroutines */
-#define SQLITE_Transitive     0x0200   /* Transitive constraints */
-#define SQLITE_OmitNoopJoin   0x0400   /* Omit unused tables in joins */
-#define SQLITE_Stat34         0x0800   /* Use STAT3 or STAT4 data */
-#define SQLITE_CountOfView    0x1000   /* The count-of-view optimization */
-#define SQLITE_CursorHints    0x2000   /* Add OP_CursorHint opcodes */
+#define SQLITE_DistinctOpt    0x0010   /* DISTINCT using indexes */
+#define SQLITE_CoverIdxScan   0x0020   /* Covering index scans */
+#define SQLITE_OrderByIdxJoin 0x0040   /* ORDER BY of joins via index */
+#define SQLITE_Transitive     0x0080   /* Transitive constraints */
+#define SQLITE_OmitNoopJoin   0x0100   /* Omit unused tables in joins */
+#define SQLITE_Stat34         0x0200   /* Use STAT3 or STAT4 data */
+#define SQLITE_CountOfView    0x0400   /* The count-of-view optimization */
+#define SQLITE_CursorHints    0x0800   /* Add OP_CursorHint opcodes */
 #define SQLITE_AllOpts        0xffff   /* All optimizations */
 
 /*
diff --git a/src/test1.c b/src/test1.c
index d70ce77a7a..446317d803 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -6901,7 +6901,6 @@ static int SQLITE_TCLAPI optimization_control(
     { "cover-idx-scan",      SQLITE_CoverIdxScan   },
     { "order-by-idx-join",   SQLITE_OrderByIdxJoin },
     { "transitive",          SQLITE_Transitive     },
-    { "subquery-coroutine",  SQLITE_SubqCoroutine  },
     { "omit-noop-join",      SQLITE_OmitNoopJoin   },
     { "stat3",               SQLITE_Stat34         },
     { "stat4",               SQLITE_Stat34         },

From 508e2d00f34f2a4d8a04130de6bf2cf44eeb1921 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 30 Sep 2017 01:25:04 +0000
Subject: [PATCH 131/270] Fix unreachable conditionals and revise a testcase
 that was made obsolete by the changes on this branch.

FossilOrigin-Name: 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/expr.c       | 11 +++++------
 src/select.c     | 27 +++++++++++++++++----------
 test/having.test | 31 ++++++++++++++++++-------------
 5 files changed, 49 insertions(+), 38 deletions(-)

diff --git a/manifest b/manifest
index 35a58bfa19..8d9fc53ff7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Always\srender\sa\ssubquery\sthat\sis\snot\spart\sof\sa\sjoin\sas\sa\sco-routine.
-D 2017-09-29T22:13:24.684
+C Fix\sunreachable\sconditionals\sand\srevise\sa\stestcase\sthat\swas\smade\sobsolete\sby\nthe\schanges\son\sthis\sbranch.
+D 2017-09-30T01:25:04.328
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -411,7 +411,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 628395aea8bbf4e2851565f9dab76565e3100ca0836a337fb748834c9cec9f04
+F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 8e8e9965af11f7ca54a44fc2cea9e8d162a6ffa1ccd2e840c69ff4468e0d08e0
+F src/select.c 55a17d54eec9093baf4ccc25342e4b2168cb8198805ed08a44fd4ae389b34b19
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -911,7 +911,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14
 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536
 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c
 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98
-F test/having.test b3d6b17cc9601b6b373b2d0f08c075ccf30e2d307249c3c8a236e3c36907b1a5
+F test/having.test e4098a4b8962f9596035c3b87a8928a10648acc509f1bb8d6f96413bbf79a1b3
 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751
 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711
 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b
-R 5c67a1a4877d581279b4ed945a250449
+P 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37
+R 3a66fd0c62efe88b91fd1bdc729b86a5
 U drh
-Z 6d48696ae0bb75b33e85df88734665c6
+Z 6d29940c14d977f820858db31a47d732
diff --git a/manifest.uuid b/manifest.uuid
index 143260d747..2da525fc6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37
\ No newline at end of file
+71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 8282c99a4a..0b922cd0fd 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1712,12 +1712,11 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
 u32 sqlite3ExprListFlags(const ExprList *pList){
   int i;
   u32 m = 0;
-  if( pList ){
-    for(i=0; i<pList->nExpr; i++){
-       Expr *pExpr = pList->a[i].pExpr;
-       assert( pExpr!=0 );
-       m |= pExpr->flags;
-    }
+  assert( pList!=0 );
+  for(i=0; i<pList->nExpr; i++){
+     Expr *pExpr = pList->a[i].pExpr;
+     assert( pExpr!=0 );
+     m |= pExpr->flags;
   }
   return m;
 }
diff --git a/src/select.c b/src/select.c
index 0f46d73f31..9c3544ae05 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3401,7 +3401,8 @@ static void substSelect(
 **        parent to a compound query confuses the code that handles
 **        recursive queries in multiSelect().
 **
-**  (24)  The subquery may not be an aggregate that uses the built-in min() or 
+**  (**)  We no longer attempt to flatten aggregate subqueries.  Was:
+**        The subquery may not be an aggregate that uses the built-in min() or 
 **        or max() functions.  (Without this restriction, a query like:
 **        "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily
 **        return the value X for which Y was maximal.)
@@ -3474,10 +3475,8 @@ static int flattenSubquery(
   if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){
      return 0;         /* Restriction (21) */
   }
-  testcase( pSub->selFlags & SF_Recursive );
-  testcase( pSub->selFlags & SF_MinMaxAgg );
-  if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){
-    return 0; /* Restrictions (22) and (24) */
+  if( pSub->selFlags & (SF_Recursive) ){
+    return 0; /* Restrictions (22) */
   }
   if( (p->selFlags & SF_Recursive) && pSub->pPrior ){
     return 0; /* Restriction (23) */
@@ -3857,14 +3856,22 @@ static int pushDownWhereTerms(
 ){
   Expr *pNew;
   int nChng = 0;
-  Select *pX;           /* For looping over compound SELECTs in pSubq */
   if( pWhere==0 ) return 0;
-  for(pX=pSubq; pX; pX=pX->pPrior){
-    if( (pX->selFlags & (SF_Recursive))!=0 ){
-      testcase( pX!=pSubq );
-      return 0; /* restriction (2) */
+  if( pSubq->selFlags & SF_Recursive ) return 0;  /* restriction (2) */
+
+#ifdef SQLITE_DEBUG
+  /* Only the first term of a compound can have a WITH clause.  But make
+  ** sure no other terms are marked SF_Recursive in case something changes
+  ** in the future.
+  */
+  {
+    Select *pX;  
+    for(pX=pSubq; pX; pX=pX->pPrior){
+      assert( (pX->selFlags & (SF_Recursive))==0 );
     }
   }
+#endif
+
   if( pSubq->pLimit!=0 ){
     return 0; /* restriction (3) */
   }
diff --git a/test/having.test b/test/having.test
index aea12319d7..a3882552d3 100644
--- a/test/having.test
+++ b/test/having.test
@@ -65,19 +65,6 @@ foreach {tn sql1 sql2} {
   3 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING a=2"
     "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a COLLATE binary"
 
-  4 {
-      SELECT x,y FROM (
-        SELECT a AS x, sum(b) AS y FROM t1 
-        GROUP BY a
-      ) WHERE x BETWEEN 8888 AND 9999
-    } {
-      SELECT x,y FROM (
-        SELECT a AS x, sum(b) AS y FROM t1 
-        WHERE x BETWEEN 8888 AND 9999 
-        GROUP BY a
-      )
-    }
-
   5 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING 0"
     "SELECT a, sum(b) FROM t1 WHERE 0 GROUP BY a COLLATE binary"
 
@@ -98,6 +85,24 @@ foreach {tn sql1 sql2} {
   do_compare_vdbe_test 2.$tn $sql1 $sql2 1
 }
 
+# The (4) test in the above set used to generate identical bytecode, but
+# that is no longer the case.  The byte code is equivalent, though.
+#
+do_execsql_test 2.4a {
+  SELECT x,y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 
+    GROUP BY a
+  ) WHERE x BETWEEN 2 AND 9999
+} {2 12}
+do_execsql_test 2.4b {
+  SELECT x,y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 
+    WHERE x BETWEEN 2 AND 9999 
+    GROUP BY a
+  )
+} {2 12}
+
+
 #-------------------------------------------------------------------------
 # 1: Test that the optimization is only applied if the GROUP BY term
 #    uses BINARY collation.

From 24ae373ab1c3e484c3013048a215a48d72a103de Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 30 Sep 2017 10:50:34 +0000
Subject: [PATCH 132/270] Make sure the SQLITE_Stat34 optimization switch is
 always 0x800, a value which is hard-coded in the TH3 test suite.

FossilOrigin-Name: 6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqliteInt.h |  7 ++++---
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 8d9fc53ff7..010e54d3d6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sunreachable\sconditionals\sand\srevise\sa\stestcase\sthat\swas\smade\sobsolete\sby\nthe\schanges\son\sthis\sbranch.
-D 2017-09-30T01:25:04.328
+C Make\ssure\sthe\sSQLITE_Stat34\soptimization\sswitch\sis\salways\s0x800,\sa\svalue\nwhich\sis\shard-coded\sin\sthe\sTH3\stest\ssuite.
+D 2017-09-30T10:50:34.309
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -464,7 +464,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h d51b1b549b6f64a44b0f5f5620aeb5bba414cf07704978238885b273ce2aa84f
+F src/sqliteInt.h 99e4beebd466495434ca6ed94bd7966a7f8c5879cd1f01768c588a5a2acd84c2
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37
-R 3a66fd0c62efe88b91fd1bdc729b86a5
+P 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e
+R abcf7527b871a7332be788888884086b
 U drh
-Z 6d29940c14d977f820858db31a47d732
+Z a8cf074d964a4716a4bd6f57e83e01aa
diff --git a/manifest.uuid b/manifest.uuid
index 2da525fc6b..6801513137 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e
\ No newline at end of file
+6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc
\ No newline at end of file
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 9b45295e70..99c8b73748 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1511,9 +1511,10 @@ struct sqlite3 {
 #define SQLITE_OrderByIdxJoin 0x0040   /* ORDER BY of joins via index */
 #define SQLITE_Transitive     0x0080   /* Transitive constraints */
 #define SQLITE_OmitNoopJoin   0x0100   /* Omit unused tables in joins */
-#define SQLITE_Stat34         0x0200   /* Use STAT3 or STAT4 data */
-#define SQLITE_CountOfView    0x0400   /* The count-of-view optimization */
-#define SQLITE_CursorHints    0x0800   /* Add OP_CursorHint opcodes */
+#define SQLITE_CountOfView    0x0200   /* The count-of-view optimization */
+#define SQLITE_CursorHints    0x0400   /* Add OP_CursorHint opcodes */
+#define SQLITE_Stat34         0x0800   /* Use STAT3 or STAT4 data */
+   /* TH3 expects the Stat34  ^^^^^^ value to be 0x0800.  Don't change it */
 #define SQLITE_AllOpts        0xffff   /* All optimizations */
 
 /*

From 67cc51a4919ae33125715e4742ce7a1ada1ad8ac Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 30 Sep 2017 11:47:06 +0000
Subject: [PATCH 133/270] More details in the header comment of
 pushDownWhereTerms().  No changes to code.

FossilOrigin-Name: 928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  6 +++++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 010e54d3d6..55d7ee5a1f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\sSQLITE_Stat34\soptimization\sswitch\sis\salways\s0x800,\sa\svalue\nwhich\sis\shard-coded\sin\sthe\sTH3\stest\ssuite.
-D 2017-09-30T10:50:34.309
+C More\sdetails\sin\sthe\sheader\scomment\sof\spushDownWhereTerms().\s\sNo\schanges\sto\ncode.
+D 2017-09-30T11:47:06.786
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 55a17d54eec9093baf4ccc25342e4b2168cb8198805ed08a44fd4ae389b34b19
+F src/select.c 19ebf10e33de8e1f327cbc844aff12af74bdf0513b3eb1b712b7bcdca4c9bdb5
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e
-R abcf7527b871a7332be788888884086b
+P 6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc
+R f601c290e8e6d8b405b953711d4d99d6
 U drh
-Z a8cf074d964a4716a4bd6f57e83e01aa
+Z 5fa1972988b6fbd3340440a07005926a
diff --git a/manifest.uuid b/manifest.uuid
index 6801513137..8a76d04632 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc
\ No newline at end of file
+928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 9c3544ae05..b7c459011c 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3831,7 +3831,11 @@ static int flattenSubquery(
 **
 **   (1) (** This restriction was removed on 2017-09-29.  We used to
 **           disallow this optimization for aggregate subqueries, but now
-**           it is allowed by putting the extra terms on the HAVING clause **)
+**           it is allowed by putting the extra terms on the HAVING clause.
+**           The added HAVING clause is pointless if the subquery lacks
+**           a GROUP BY clause.  But such a HAVING clause is also harmless
+**           so there does not appear to be any reason to add extra logic
+**           to suppress it. **)
 **
 **   (2) The inner query is the recursive part of a common table expression.
 **

From de6422a609ea9fa7fade846acc874100a7361b7e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 2 Oct 2017 02:32:12 +0000
Subject: [PATCH 134/270] Improvements to a comment in lempar.c.  No code
 changes.

FossilOrigin-Name: 8000d230171fe53599fc6a916666f60f88f789e6c9c6c2869514c02d837d6fc8
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 87b5990a98..3dc920122d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clean\sup\sthe\scomments\son\sthe\squery\sflattener\sto\smore\sclearly\sand\saccurately\nexpress\sthe\sconditions\sunder\swhich\sthe\sflattener\sis\sable\sto\srun.
-D 2017-09-29T16:07:56.167
+C Improvements\sto\sa\scomment\sin\slempar.c.\s\sNo\scode\schanges.
+D 2017-10-02T02:32:12.306
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1579,7 +1579,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c 10579a61dc2290182725e7abdefe311dd8b521a8f7f0aabbfc571e9012a09eaf
+F tool/lempar.c 105d0d9cbe5a25d24d4769241ffbfc63ac7c09e6ccee0dc43dcc8a4c4ae4e426
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2a45bbc9fd1c64f1c4c4dac38f063cd67480fcb6da24bf93fdefbfca66fd81ab
-R 5dd9d1abad7c6b3045de562026df2e52
+P 0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86
+R 7f732cb0e00eb0e6e46dbdc3b44db828
 U drh
-Z 40508487cfccebe9fe70dcdbc46f56d0
+Z bc9e28c24d7165521312a05b1d8d5118
diff --git a/manifest.uuid b/manifest.uuid
index 48e1728350..254a13150d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86
\ No newline at end of file
+8000d230171fe53599fc6a916666f60f88f789e6c9c6c2869514c02d837d6fc8
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 03e30bd164..37a5892195 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -75,7 +75,8 @@
 **    YY_MAX_SHIFT       Maximum value for shift actions
 **    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
 **    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
-**    YY_MIN_REDUCE      Maximum value for reduce actions
+**    YY_MIN_REDUCE      Minimum value for reduce actions
+**    YY_MAX_REDUCE      Maximum value for reduce actions
 **    YY_ERROR_ACTION    The yy_action[] code for syntax error
 **    YY_ACCEPT_ACTION   The yy_action[] code for accept
 **    YY_NO_ACTION       The yy_action[] code for no-op

From 3ee1416b68736da726ccf61def18d371ebcff793 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 2 Oct 2017 13:20:43 +0000
Subject: [PATCH 135/270] Avoid using lookaside memory for persistent virtual
 table structures.

FossilOrigin-Name: d815f255dfbcd6d64326f7bc0ad3fe3c4ff08270ca75f8836ef2a919d5e57401
---
 manifest      | 13 ++++++-------
 manifest.uuid |  2 +-
 src/vtab.c    | 11 +++++++----
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 4f79804d82..43948ec62f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\squery\splanner\snow\salways\sprefers\sto\simplement\sa\sFROM-clause\ssubquery\susing\na\sco-routine\srather\sthan\sflattening\sthe\ssubquery\sinto\sthe\souter\squery.
-D 2017-10-02T02:52:54.153
+C Avoid\susing\slookaside\smemory\sfor\spersistent\svirtual\stable\sstructures.
+D 2017-10-02T13:20:43.199
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -535,7 +535,7 @@ F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c028
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
-F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
+F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
@@ -1655,8 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8000d230171fe53599fc6a916666f60f88f789e6c9c6c2869514c02d837d6fc8 928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689
-R d80162b36b4fccf5799907b67fd02d3c
-T +closed 928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689
+P c9104b59c7ed360291f7f6fc8caae938e9840c77620d598e4096f78183bf807a
+R 4fb59e163271c3c6913da8e89cf04532
 U drh
-Z dc6d533f9eb613034e3c381c85c3f10b
+Z b5205318031b18ac166280920d64572f
diff --git a/manifest.uuid b/manifest.uuid
index 9d275448b7..ef6a93ee5d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c9104b59c7ed360291f7f6fc8caae938e9840c77620d598e4096f78183bf807a
\ No newline at end of file
+d815f255dfbcd6d64326f7bc0ad3fe3c4ff08270ca75f8836ef2a919d5e57401
\ No newline at end of file
diff --git a/src/vtab.c b/src/vtab.c
index 634dccc03c..bc1fa3e8cc 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -42,8 +42,10 @@ Module *sqlite3VtabCreateModule(
 ){
   Module *pMod;
   int nName = sqlite3Strlen30(zName);
-  pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1);
-  if( pMod ){
+  pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1);
+  if( pMod==0 ){
+    sqlite3OomFault(db);
+  }else{
     Module *pDel;
     char *zCopy = (char *)(&pMod[1]);
     memcpy(zCopy, zName, nName+1);
@@ -518,13 +520,14 @@ static int vtabCallConstructor(
     }
   }
 
-  zModuleName = sqlite3MPrintf(db, "%s", pTab->zName);
+  zModuleName = sqlite3DbStrDup(db, pTab->zName);
   if( !zModuleName ){
     return SQLITE_NOMEM_BKPT;
   }
 
-  pVTable = sqlite3DbMallocZero(db, sizeof(VTable));
+  pVTable = sqlite3MallocZero(sizeof(VTable));
   if( !pVTable ){
+    sqlite3OomFault(db);
     sqlite3DbFree(db, zModuleName);
     return SQLITE_NOMEM_BKPT;
   }

From 2f65b2f512ef02d5daf703a870b713534796ea56 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 2 Oct 2017 21:29:51 +0000
Subject: [PATCH 136/270] Add the OK_IF_ALWAYS_TRUE() and OK_IF_ALWAYS_FALSE()
 macros for marking conditionals that improve performance but do not change
 the outcome.

FossilOrigin-Name: 6035c9b2728f47d338696978eb6fe5a7d6cb84bbea0792ef985c0986ac8f39dd
---
 manifest             | 16 ++++++++--------
 manifest.uuid        |  2 +-
 src/select.c         |  4 ++--
 src/sqliteInt.h      | 15 +++++++++++++++
 test/releasetest.tcl |  1 +
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 43948ec62f..c0fed11f6c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\susing\slookaside\smemory\sfor\spersistent\svirtual\stable\sstructures.
-D 2017-10-02T13:20:43.199
+C Add\sthe\sOK_IF_ALWAYS_TRUE()\sand\sOK_IF_ALWAYS_FALSE()\smacros\sfor\smarking\nconditionals\sthat\simprove\sperformance\sbut\sdo\snot\schange\sthe\soutcome.
+D 2017-10-02T21:29:51.437
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,13 +458,13 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 19ebf10e33de8e1f327cbc844aff12af74bdf0513b3eb1b712b7bcdca4c9bdb5
+F src/select.c 38ab76414b86e98dfa5b2fbec695c97760ccb242ee3aa275fc754fad7232b9a2
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 99e4beebd466495434ca6ed94bd7966a7f8c5879cd1f01768c588a5a2acd84c2
+F src/sqliteInt.h 50a509839bae7d27d1a09fe4e74432baa1d63f6e1b39383546cf461721c3572a
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1122,7 +1122,7 @@ F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
-F test/releasetest.tcl 22bd6be9f227a1ad01ebfaa8ed92400b5eaaff1ecc1e14e269eec2631c7f43ab x
+F test/releasetest.tcl 0b0b3d926e36822ff63b405d683544ce1014303b029f2678bbcf40c162b5f246 x
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test f580934279800d480a19176c6b44909df31ce7ad45267ea475a541daa522f3d3
 F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c9104b59c7ed360291f7f6fc8caae938e9840c77620d598e4096f78183bf807a
-R 4fb59e163271c3c6913da8e89cf04532
+P d815f255dfbcd6d64326f7bc0ad3fe3c4ff08270ca75f8836ef2a919d5e57401
+R cad5ba00a1afaf9521996fdcf47b3324
 U drh
-Z b5205318031b18ac166280920d64572f
+Z d8ed85ff60322b19e2b85a6468ce5a8d
diff --git a/manifest.uuid b/manifest.uuid
index ef6a93ee5d..df174ee5ea 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d815f255dfbcd6d64326f7bc0ad3fe3c4ff08270ca75f8836ef2a919d5e57401
\ No newline at end of file
+6035c9b2728f47d338696978eb6fe5a7d6cb84bbea0792ef985c0986ac8f39dd
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index b7c459011c..3f98daacee 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4285,7 +4285,7 @@ static int withExpand(
 */
 static void selectPopWith(Walker *pWalker, Select *p){
   Parse *pParse = pWalker->pParse;
-  if( pParse->pWith && p->pPrior==0 ){
+  if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){
     With *pWith = findRightmost(p)->pWith;
     if( pWith!=0 ){
       assert( pParse->pWith==pWith );
@@ -4340,7 +4340,7 @@ static int selectExpander(Walker *pWalker, Select *p){
   }
   pTabList = p->pSrc;
   pEList = p->pEList;
-  if( p->pWith ){
+  if( OK_IF_ALWAYS_TRUE(p->pWith) ){
     sqlite3WithPush(pParse, p->pWith, 0);
   }
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 99c8b73748..7c39c7b919 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -446,6 +446,21 @@
 # define NEVER(X)       (X)
 #endif
 
+/*
+** Some conditionals are optimizations only.  In other words, if the
+** conditionals are replaced with a constant 1 (true) or 0 (false) then
+** the correct answer is still obtained, though perhaps not as quickly.
+**
+** The following macros mark these optimizations conditionals.
+*/
+#if defined(SQLITE_MUTATION_TEST)
+# define OK_IF_ALWAYS_TRUE(X)  (1)
+# define OK_IF_ALWAYS_FALSE(X) (0)
+#else
+# define OK_IF_ALWAYS_TRUE(X)  (X)
+# define OK_IF_ALWAYS_FALSE(X) (X)
+#endif
+
 /*
 ** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is
 ** defined.  We need to defend against those failures when testing with
diff --git a/test/releasetest.tcl b/test/releasetest.tcl
index d5f758274d..5055e81129 100755
--- a/test/releasetest.tcl
+++ b/test/releasetest.tcl
@@ -126,6 +126,7 @@ array set ::Configs [strip_comments {
     -DSQLITE_ENABLE_STAT4
     -DSQLITE_ENABLE_HIDDEN_COLUMNS
     -DSQLITE_MAX_ATTACHED=125
+    -DSQLITE_MUTATION_TEST
   }
   "Fast-One" {
     -O6

From b7651e6b60b08a87683462aac14f11b70f2c8855 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 2 Oct 2017 23:30:45 +0000
Subject: [PATCH 137/270] Simplification to sqlite3SelectPrep() for improved
 testability.

FossilOrigin-Name: 5bf2ccb908dc6ebf7e063014b0a4bed7e3e9e0a555f78c6b4f6647adfb7209e4
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  9 +++------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index c0fed11f6c..23f302f630 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sOK_IF_ALWAYS_TRUE()\sand\sOK_IF_ALWAYS_FALSE()\smacros\sfor\smarking\nconditionals\sthat\simprove\sperformance\sbut\sdo\snot\schange\sthe\soutcome.
-D 2017-10-02T21:29:51.437
+C Simplification\sto\ssqlite3SelectPrep()\sfor\simproved\stestability.
+D 2017-10-02T23:30:45.503
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 38ab76414b86e98dfa5b2fbec695c97760ccb242ee3aa275fc754fad7232b9a2
+F src/select.c c12d9b45c31182921c2aea5ab11bc0add480c6e3b47cb3400edecbbf97000fd0
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d815f255dfbcd6d64326f7bc0ad3fe3c4ff08270ca75f8836ef2a919d5e57401
-R cad5ba00a1afaf9521996fdcf47b3324
+P 6035c9b2728f47d338696978eb6fe5a7d6cb84bbea0792ef985c0986ac8f39dd
+R 7f0bb82bff2a71c21175e4e21944fba8
 U drh
-Z d8ed85ff60322b19e2b85a6468ce5a8d
+Z edd3956faff3d872b10136acafe045b5
diff --git a/manifest.uuid b/manifest.uuid
index df174ee5ea..1f15413459 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6035c9b2728f47d338696978eb6fe5a7d6cb84bbea0792ef985c0986ac8f39dd
\ No newline at end of file
+5bf2ccb908dc6ebf7e063014b0a4bed7e3e9e0a555f78c6b4f6647adfb7209e4
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 3f98daacee..cefd2c27f5 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4733,15 +4733,12 @@ void sqlite3SelectPrep(
   Select *p,             /* The SELECT statement being coded. */
   NameContext *pOuterNC  /* Name context for container */
 ){
-  sqlite3 *db;
-  if( NEVER(p==0) ) return;
-  db = pParse->db;
-  if( db->mallocFailed ) return;
+  assert( p!=0 );
   if( p->selFlags & SF_HasTypeInfo ) return;
   sqlite3SelectExpand(pParse, p);
-  if( pParse->nErr || db->mallocFailed ) return;
+  if( pParse->nErr || pParse->db->mallocFailed ) return;
   sqlite3ResolveSelectNames(pParse, p, pOuterNC);
-  if( pParse->nErr || db->mallocFailed ) return;
+  if( pParse->nErr || pParse->db->mallocFailed ) return;
   sqlite3SelectAddTypeInfo(pParse, p);
 }
 

From 878fcf9d4ef72b8cf2eebd79566ac6ccf9f191f6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 2 Oct 2017 23:50:08 +0000
Subject: [PATCH 138/270] Mark a single branch within sqlite3SelectExpand() as
 an optimization.

FossilOrigin-Name: 3a4ffb21d0ee2326c31b9ddc512d79d4ebf3a249a2840e88e99165978eec31fd
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 23f302f630..4d07269409 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sto\ssqlite3SelectPrep()\sfor\simproved\stestability.
-D 2017-10-02T23:30:45.503
+C Mark\sa\ssingle\sbranch\swithin\ssqlite3SelectExpand()\sas\san\soptimization.
+D 2017-10-02T23:50:08.350
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c c12d9b45c31182921c2aea5ab11bc0add480c6e3b47cb3400edecbbf97000fd0
+F src/select.c 3e3fb2e45e26fe5f12d1765e6c40c57762977f36a2436aaa7463b82eb5ed760f
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6035c9b2728f47d338696978eb6fe5a7d6cb84bbea0792ef985c0986ac8f39dd
-R 7f0bb82bff2a71c21175e4e21944fba8
+P 5bf2ccb908dc6ebf7e063014b0a4bed7e3e9e0a555f78c6b4f6647adfb7209e4
+R eefaca2ea58aad260c29aab3ab787ea1
 U drh
-Z edd3956faff3d872b10136acafe045b5
+Z 1bd2e271ce965a5407b85612faa9a389
diff --git a/manifest.uuid b/manifest.uuid
index 1f15413459..c98c948424 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5bf2ccb908dc6ebf7e063014b0a4bed7e3e9e0a555f78c6b4f6647adfb7209e4
\ No newline at end of file
+3a4ffb21d0ee2326c31b9ddc512d79d4ebf3a249a2840e88e99165978eec31fd
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index cefd2c27f5..8a14920084 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4645,7 +4645,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
   Walker w;
   w.xExprCallback = sqlite3ExprWalkNoop;
   w.pParse = pParse;
-  if( pParse->hasCompound ){
+  if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){
     w.xSelectCallback = convertCompoundSelectToSubquery;
     w.xSelectCallback2 = 0;
     sqlite3WalkSelect(&w, pSelect);

From cafc2f7b3ecf9895dcc78a053b10224d2e6898d9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 03:01:09 +0000
Subject: [PATCH 139/270] Do not compute column and row size estimates for the
 transient Table objects associated with the result set of a SELECT statement,
 since those estimates are never used for anything constructive.

FossilOrigin-Name: e4342fd401d4025eecc53ebfcd59f3150166a5f0bd6adb1b5820703bd4a1ec58
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 30 +++++++++++++-----------------
 3 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index 4d07269409..7c8cd81854 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Mark\sa\ssingle\sbranch\swithin\ssqlite3SelectExpand()\sas\san\soptimization.
-D 2017-10-02T23:50:08.350
+C Do\snot\scompute\scolumn\sand\srow\ssize\sestimates\sfor\sthe\stransient\sTable\sobjects\nassociated\swith\sthe\sresult\sset\sof\sa\sSELECT\sstatement,\ssince\sthose\sestimates\nare\snever\sused\sfor\sanything\sconstructive.
+D 2017-10-03T03:01:09.197
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 3e3fb2e45e26fe5f12d1765e6c40c57762977f36a2436aaa7463b82eb5ed760f
+F src/select.c 1a11e6ab321c22a20146c44d315ece07899f7abad5dab839c9d4cf6556ef588b
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5bf2ccb908dc6ebf7e063014b0a4bed7e3e9e0a555f78c6b4f6647adfb7209e4
-R eefaca2ea58aad260c29aab3ab787ea1
+P 3a4ffb21d0ee2326c31b9ddc512d79d4ebf3a249a2840e88e99165978eec31fd
+R 2ffb7cd4a155ee43580067e1a47b5521
 U drh
-Z 1bd2e271ce965a5407b85612faa9a389
+Z 7536be1ae6bc2716a9cf67864c3578f3
diff --git a/manifest.uuid b/manifest.uuid
index c98c948424..0fc4d2a1df 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3a4ffb21d0ee2326c31b9ddc512d79d4ebf3a249a2840e88e99165978eec31fd
\ No newline at end of file
+e4342fd401d4025eecc53ebfcd59f3150166a5f0bd6adb1b5820703bd4a1ec58
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 8a14920084..c77e14a832 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1362,23 +1362,23 @@ static void generateSortTail(
 ** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used.
 */
 #ifdef SQLITE_ENABLE_COLUMN_METADATA
-# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F)
+# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E)
 #else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */
-# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F)
+# define columnType(A,B,C,D,E) columnTypeImpl(A,B)
 #endif
 static const char *columnTypeImpl(
   NameContext *pNC, 
+#ifndef SQLITE_ENABLE_COLUMN_METADATA
+  Expr *pExpr
+#else
   Expr *pExpr,
-#ifdef SQLITE_ENABLE_COLUMN_METADATA
   const char **pzOrigDb,
   const char **pzOrigTab,
-  const char **pzOrigCol,
+  const char **pzOrigCol
 #endif
-  u8 *pEstWidth
 ){
   char const *zType = 0;
   int j;
-  u8 estWidth = 1;
 #ifdef SQLITE_ENABLE_COLUMN_METADATA
   char const *zOrigDb = 0;
   char const *zOrigTab = 0;
@@ -1447,7 +1447,7 @@ static const char *columnTypeImpl(
           sNC.pSrcList = pS->pSrc;
           sNC.pNext = pNC;
           sNC.pParse = pNC->pParse;
-          zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); 
+          zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); 
         }
       }else if( pTab->pSchema ){
         /* A real table */
@@ -1461,7 +1461,6 @@ static const char *columnTypeImpl(
         }else{
           zOrigCol = pTab->aCol[iCol].zName;
           zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
-          estWidth = pTab->aCol[iCol].szEst;
         }
         zOrigTab = pTab->zName;
         if( pNC->pParse ){
@@ -1473,7 +1472,6 @@ static const char *columnTypeImpl(
           zType = "INTEGER";
         }else{
           zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
-          estWidth = pTab->aCol[iCol].szEst;
         }
 #endif
       }
@@ -1492,7 +1490,7 @@ static const char *columnTypeImpl(
       sNC.pSrcList = pS->pSrc;
       sNC.pNext = pNC;
       sNC.pParse = pNC->pParse;
-      zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, &estWidth); 
+      zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); 
       break;
     }
 #endif
@@ -1506,7 +1504,6 @@ static const char *columnTypeImpl(
     *pzOrigCol = zOrigCol;
   }
 #endif
-  if( pEstWidth ) *pEstWidth = estWidth;
   return zType;
 }
 
@@ -1533,7 +1530,7 @@ static void generateColumnTypes(
     const char *zOrigDb = 0;
     const char *zOrigTab = 0;
     const char *zOrigCol = 0;
-    zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, 0);
+    zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol);
 
     /* The vdbe must make its own copy of the column-type and other 
     ** column specific strings, in case the schema is reset before this
@@ -1543,7 +1540,7 @@ static void generateColumnTypes(
     sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT);
     sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT);
 #else
-    zType = columnType(&sNC, p, 0, 0, 0, 0);
+    zType = columnType(&sNC, p, 0, 0, 0);
 #endif
     sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT);
   }
@@ -1788,7 +1785,6 @@ void sqlite3SelectAddColumnTypeAndCollation(
   int i;
   Expr *p;
   struct ExprList_item *a;
-  u64 szAll = 0;
 
   assert( pSelect!=0 );
   assert( (pSelect->selFlags & SF_Resolved)!=0 );
@@ -1801,8 +1797,8 @@ void sqlite3SelectAddColumnTypeAndCollation(
     const char *zType;
     int n, m;
     p = a[i].pExpr;
-    zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst);
-    szAll += pCol->szEst;
+    zType = columnType(&sNC, p, 0, 0, 0);
+    /* pCol->szEst = ... // Column size est for SELECT tables never used */
     pCol->affinity = sqlite3ExprAffinity(p);
     if( zType && (m = sqlite3Strlen30(zType))>0 ){
       n = sqlite3Strlen30(pCol->zName);
@@ -1818,7 +1814,7 @@ void sqlite3SelectAddColumnTypeAndCollation(
       pCol->zColl = sqlite3DbStrDup(db, pColl->zName);
     }
   }
-  pTab->szTabRow = sqlite3LogEst(szAll*4);
+  pTab->szTabRow = 1; /* Any non-zero value works */
 }
 
 /*

From e2463398fd1154581c078347b5000201b54f037e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 14:24:24 +0000
Subject: [PATCH 140/270] The simplification to sqlite3SelectExpand() in last
 night's [3a4ffb21] check-in was not completely correct.  This adjustment is
 needed for correct OOM handling in some configurations.

FossilOrigin-Name: c5ad5e1675f6cb5a54df21606dfec2198a7cf88acc529460ebe27bdab128a6e5
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7c8cd81854..0c2ed835c9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\scompute\scolumn\sand\srow\ssize\sestimates\sfor\sthe\stransient\sTable\sobjects\nassociated\swith\sthe\sresult\sset\sof\sa\sSELECT\sstatement,\ssince\sthose\sestimates\nare\snever\sused\sfor\sanything\sconstructive.
-D 2017-10-03T03:01:09.197
+C The\ssimplification\sto\ssqlite3SelectExpand()\sin\slast\snight's\n[3a4ffb21]\scheck-in\swas\snot\scompletely\scorrect.\s\sThis\sadjustment\sis\sneeded\nfor\scorrect\sOOM\shandling\sin\ssome\sconfigurations.
+D 2017-10-03T14:24:24.678
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 1a11e6ab321c22a20146c44d315ece07899f7abad5dab839c9d4cf6556ef588b
+F src/select.c 948aa57117275e1f3c6713254ec2baae09be058bec40ab7dcc41960202b756e5
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3a4ffb21d0ee2326c31b9ddc512d79d4ebf3a249a2840e88e99165978eec31fd
-R 2ffb7cd4a155ee43580067e1a47b5521
+P e4342fd401d4025eecc53ebfcd59f3150166a5f0bd6adb1b5820703bd4a1ec58
+R 067a132a78e0ff9232176facb949a1bf
 U drh
-Z 7536be1ae6bc2716a9cf67864c3578f3
+Z ad07329ae2092cc1b6846d3db3cadef4
diff --git a/manifest.uuid b/manifest.uuid
index 0fc4d2a1df..6176434c11 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e4342fd401d4025eecc53ebfcd59f3150166a5f0bd6adb1b5820703bd4a1ec58
\ No newline at end of file
+c5ad5e1675f6cb5a54df21606dfec2198a7cf88acc529460ebe27bdab128a6e5
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index c77e14a832..8109408697 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4729,7 +4729,8 @@ void sqlite3SelectPrep(
   Select *p,             /* The SELECT statement being coded. */
   NameContext *pOuterNC  /* Name context for container */
 ){
-  assert( p!=0 );
+  assert( p!=0 || pParse->db->mallocFailed );
+  if( pParse->db->mallocFailed ) return;
   if( p->selFlags & SF_HasTypeInfo ) return;
   sqlite3SelectExpand(pParse, p);
   if( pParse->nErr || pParse->db->mallocFailed ) return;

From a78d757c36f300ba0fdfefb11ae4a50ee8c0ccc0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 16:57:33 +0000
Subject: [PATCH 141/270] Fix the sqlite3_column_decltype() interface to return
 the correct datatype when the value comes through a CTE.

FossilOrigin-Name: 966438bd259ade3a2d4a6ec9587ce79c81a21b942248bffb389995b9132b53e3
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 11 ++++++-----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 0c2ed835c9..0026fbbe1f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\ssimplification\sto\ssqlite3SelectExpand()\sin\slast\snight's\n[3a4ffb21]\scheck-in\swas\snot\scompletely\scorrect.\s\sThis\sadjustment\sis\sneeded\nfor\scorrect\sOOM\shandling\sin\ssome\sconfigurations.
-D 2017-10-03T14:24:24.678
+C Fix\sthe\ssqlite3_column_decltype()\sinterface\sto\sreturn\sthe\scorrect\sdatatype\nwhen\sthe\svalue\scomes\sthrough\sa\sCTE.
+D 2017-10-03T16:57:33.183
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 948aa57117275e1f3c6713254ec2baae09be058bec40ab7dcc41960202b756e5
+F src/select.c cb912fc73ee6dd404acdee580ee3bcea593e6b1f068c539d4298b7010e664c61
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e4342fd401d4025eecc53ebfcd59f3150166a5f0bd6adb1b5820703bd4a1ec58
-R 067a132a78e0ff9232176facb949a1bf
+P c5ad5e1675f6cb5a54df21606dfec2198a7cf88acc529460ebe27bdab128a6e5
+R 4e08b893abd2a9bcdfb81ab1fd1cc1bd
 U drh
-Z ad07329ae2092cc1b6846d3db3cadef4
+Z 94cd089961159ef74793b517e37fdcdc
diff --git a/manifest.uuid b/manifest.uuid
index 6176434c11..42b7a639a2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c5ad5e1675f6cb5a54df21606dfec2198a7cf88acc529460ebe27bdab128a6e5
\ No newline at end of file
+966438bd259ade3a2d4a6ec9587ce79c81a21b942248bffb389995b9132b53e3
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 8109408697..75ba752d4c 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1449,12 +1449,12 @@ static const char *columnTypeImpl(
           sNC.pParse = pNC->pParse;
           zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); 
         }
-      }else if( pTab->pSchema ){
-        /* A real table */
+      }else{
+        /* A real table or a CTE table */
         assert( !pS );
-        if( iCol<0 ) iCol = pTab->iPKey;
-        assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
 #ifdef SQLITE_ENABLE_COLUMN_METADATA
+        if( iCol<0 ) iCol = pTab->iPKey;
+        assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
         if( iCol<0 ){
           zType = "INTEGER";
           zOrigCol = "rowid";
@@ -1463,11 +1463,12 @@ static const char *columnTypeImpl(
           zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
         }
         zOrigTab = pTab->zName;
-        if( pNC->pParse ){
+        if( pNC->pParse && pTab->pSchema ){
           int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema);
           zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName;
         }
 #else
+        assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
         if( iCol<0 ){
           zType = "INTEGER";
         }else{

From ce2c482e5a3876c7ad9f594ef3f771e2ca24b058 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 17:17:34 +0000
Subject: [PATCH 142/270] Remove an unused variable from sqlite3Insert() and
 fix harmless compiler warnings associated with -DSQLITE_MUTATION_TEST.

FossilOrigin-Name: 7be760e907274131bcd4acfaff9e72c9c59a05e7d411db3d9afe690fe0d64b4b
---
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 src/insert.c  |  3 ---
 src/main.c    |  4 ++--
 src/select.c  |  3 +--
 5 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 0026fbbe1f..64e4df70a2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\ssqlite3_column_decltype()\sinterface\sto\sreturn\sthe\scorrect\sdatatype\nwhen\sthe\svalue\scomes\sthrough\sa\sCTE.
-D 2017-10-03T16:57:33.183
+C Remove\san\sunused\svariable\sfrom\ssqlite3Insert()\sand\sfix\sharmless\ncompiler\swarnings\sassociated\swith\s-DSQLITE_MUTATION_TEST.
+D 2017-10-03T17:17:34.912
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -420,10 +420,10 @@ F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c d2d1bf12d2b5382450620d7cede84c7ffe57e6a89fa9a908f1aba68df2731cd9
+F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 1c2307a9f7e3183232a2d049881026fe43bf25659857cc11a9a44898accd3200
+F src/main.c 556a0942bdab9cd33f75aaec4adf54cc008898cb73b9657337be5960c7bcac8f
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c cb912fc73ee6dd404acdee580ee3bcea593e6b1f068c539d4298b7010e664c61
+F src/select.c 52f5ef16847ace5f44cb05474b18aa168ce4191ab216f1faaeb27100c588933a
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c5ad5e1675f6cb5a54df21606dfec2198a7cf88acc529460ebe27bdab128a6e5
-R 4e08b893abd2a9bcdfb81ab1fd1cc1bd
+P 966438bd259ade3a2d4a6ec9587ce79c81a21b942248bffb389995b9132b53e3
+R ffa559a096a20092f6ad3c894c3f67a3
 U drh
-Z 94cd089961159ef74793b517e37fdcdc
+Z 7ef5ddb828d35332c3570a426acb47a9
diff --git a/manifest.uuid b/manifest.uuid
index 42b7a639a2..1780765a08 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-966438bd259ade3a2d4a6ec9587ce79c81a21b942248bffb389995b9132b53e3
\ No newline at end of file
+7be760e907274131bcd4acfaff9e72c9c59a05e7d411db3d9afe690fe0d64b4b
\ No newline at end of file
diff --git a/src/insert.c b/src/insert.c
index f8e9095ea5..5da52e7692 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -484,7 +484,6 @@ void sqlite3Insert(
 ){
   sqlite3 *db;          /* The main database structure */
   Table *pTab;          /* The table to insert into.  aka TABLE */
-  char *zTab;           /* Name of the table into which we are inserting */
   int i, j;             /* Loop counters */
   Vdbe *v;              /* Generate code into this virtual machine */
   Index *pIdx;          /* For looping over indices of the table */
@@ -540,8 +539,6 @@ void sqlite3Insert(
   /* Locate the table into which we will be inserting new information.
   */
   assert( pTabList->nSrc==1 );
-  zTab = pTabList->a[0].zName;
-  if( NEVER(zTab==0) ) goto insert_cleanup;
   pTab = sqlite3SrcListLookup(pParse, pTabList);
   if( pTab==0 ){
     goto insert_cleanup;
diff --git a/src/main.c b/src/main.c
index 6f697ffb39..082b96f960 100644
--- a/src/main.c
+++ b/src/main.c
@@ -3711,7 +3711,7 @@ int sqlite3_test_control(int op, ...){
     ** This action provides a run-time test to see how the ALWAYS and
     ** NEVER macros were defined at compile-time.
     **
-    ** The return value is ALWAYS(X).  
+    ** The return value is ALWAYS(X) if X is true, or 0 if X is false.
     **
     ** The recommended test is X==2.  If the return value is 2, that means
     ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the
@@ -3734,7 +3734,7 @@ int sqlite3_test_control(int op, ...){
     */
     case SQLITE_TESTCTRL_ALWAYS: {
       int x = va_arg(ap,int);
-      rc = ALWAYS(x);
+      rc = x ? ALWAYS(x) : 0;
       break;
     }
 
diff --git a/src/select.c b/src/select.c
index 75ba752d4c..c279472f33 100644
--- a/src/select.c
+++ b/src/select.c
@@ -412,11 +412,10 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){
   pLeft = &pSrc->a[0];
   pRight = &pLeft[1];
   for(i=0; i<pSrc->nSrc-1; i++, pRight++, pLeft++){
-    Table *pLeftTab = pLeft->pTab;
     Table *pRightTab = pRight->pTab;
     int isOuter;
 
-    if( NEVER(pLeftTab==0 || pRightTab==0) ) continue;
+    if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue;
     isOuter = (pRight->fg.jointype & JT_OUTER)!=0;
 
     /* When the NATURAL keyword is present, add WHERE clause terms for

From 8906a4b80258cd0713e100150b94991888b0db96 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 17:29:40 +0000
Subject: [PATCH 143/270] Add some OK_IF_ALWAYS_TRUE() marks on optimization
 branches in select.c.

FossilOrigin-Name: 1dd828088d6981dfebf9f4d650dad8431bece4405650c61f90eb8d8f43289b52
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 64e4df70a2..a1be774e26 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunused\svariable\sfrom\ssqlite3Insert()\sand\sfix\sharmless\ncompiler\swarnings\sassociated\swith\s-DSQLITE_MUTATION_TEST.
-D 2017-10-03T17:17:34.912
+C Add\ssome\sOK_IF_ALWAYS_TRUE()\smarks\son\soptimization\sbranches\sin\sselect.c.
+D 2017-10-03T17:29:40.582
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 52f5ef16847ace5f44cb05474b18aa168ce4191ab216f1faaeb27100c588933a
+F src/select.c 764d5ac2bc4d96a9183d8ee3067f19ef4133aa84788d89875e9626f906703171
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 966438bd259ade3a2d4a6ec9587ce79c81a21b942248bffb389995b9132b53e3
-R ffa559a096a20092f6ad3c894c3f67a3
+P 7be760e907274131bcd4acfaff9e72c9c59a05e7d411db3d9afe690fe0d64b4b
+R ce19fff627af42bc467b7af644b3b595
 U drh
-Z 7ef5ddb828d35332c3570a426acb47a9
+Z ecfbf86cbb40553da9b0819689d116ec
diff --git a/manifest.uuid b/manifest.uuid
index 1780765a08..d144e21c94 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7be760e907274131bcd4acfaff9e72c9c59a05e7d411db3d9afe690fe0d64b4b
\ No newline at end of file
+1dd828088d6981dfebf9f4d650dad8431bece4405650c61f90eb8d8f43289b52
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index c279472f33..f628e56f64 100644
--- a/src/select.c
+++ b/src/select.c
@@ -75,7 +75,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){
     sqlite3ExprListDelete(db, p->pOrderBy);
     sqlite3ExprDelete(db, p->pLimit);
     sqlite3ExprDelete(db, p->pOffset);
-    if( p->pWith ) sqlite3WithDelete(db, p->pWith);
+    if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith);
     if( bFree ) sqlite3DbFreeNN(db, p);
     p = pPrior;
     bFree = 1;
@@ -171,7 +171,7 @@ void sqlite3SelectSetName(Select *p, const char *zName){
 ** Delete the given Select structure and all of its substructures.
 */
 void sqlite3SelectDelete(sqlite3 *db, Select *p){
-  if( p ) clearSelect(db, p, 1);
+  if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
 }
 
 /*

From 997a4deb4edd780d2e491fcab120ac9bba5441b6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 18:35:57 +0000
Subject: [PATCH 144/270] Do not enable SELECT tracing with -DSQLITE_DEBUG. 
 Require the -DSQLITE_ENABLE_SELECTTRACE compile-time option to enable SELECT
 tracing.

FossilOrigin-Name: f5c395834c2a776beba6fe172cc4a5e428ce30b9cb4259ef8e440f10455a41e9
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqliteInt.h |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index a1be774e26..425c460246 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssome\sOK_IF_ALWAYS_TRUE()\smarks\son\soptimization\sbranches\sin\sselect.c.
-D 2017-10-03T17:29:40.582
+C Do\snot\senable\sSELECT\stracing\swith\s-DSQLITE_DEBUG.\s\sRequire\sthe\n-DSQLITE_ENABLE_SELECTTRACE\scompile-time\soption\sto\senable\sSELECT\stracing.
+D 2017-10-03T18:35:57.160
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -464,7 +464,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h 50a509839bae7d27d1a09fe4e74432baa1d63f6e1b39383546cf461721c3572a
+F src/sqliteInt.h c07bc88eca1f59ce73e1f486187d0df4effe67c4579e112dfdd91c159e5c0569
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7be760e907274131bcd4acfaff9e72c9c59a05e7d411db3d9afe690fe0d64b4b
-R ce19fff627af42bc467b7af644b3b595
+P 1dd828088d6981dfebf9f4d650dad8431bece4405650c61f90eb8d8f43289b52
+R ff9a44814ead03fb339e5473f4e89eb3
 U drh
-Z ecfbf86cbb40553da9b0819689d116ec
+Z 7ee7c6afb266c430d526e6cc80b0cc29
diff --git a/manifest.uuid b/manifest.uuid
index d144e21c94..bbdc43be85 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1dd828088d6981dfebf9f4d650dad8431bece4405650c61f90eb8d8f43289b52
\ No newline at end of file
+f5c395834c2a776beba6fe172cc4a5e428ce30b9cb4259ef8e440f10455a41e9
\ No newline at end of file
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 7c39c7b919..a8f1bed512 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -944,7 +944,7 @@ typedef INT16_TYPE LogEst;
 ** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not
 ** the Select query generator tracing logic is turned on.
 */
-#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE)
+#if defined(SQLITE_ENABLE_SELECTTRACE)
 # define SELECTTRACE_ENABLED 1
 #else
 # define SELECTTRACE_ENABLED 0

From 0c4db03481790098974207a6c2e9e551bb3fc3e7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 3 Oct 2017 19:53:12 +0000
Subject: [PATCH 145/270] Simplify the computation of types on columns of a
 view.

FossilOrigin-Name: 772b0db1469c9e1c2728cf65dd070e29c624e75c3a5da72ee1297f15d758dd13
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 425c460246..b9988d900e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\senable\sSELECT\stracing\swith\s-DSQLITE_DEBUG.\s\sRequire\sthe\n-DSQLITE_ENABLE_SELECTTRACE\scompile-time\soption\sto\senable\sSELECT\stracing.
-D 2017-10-03T18:35:57.160
+C Simplify\sthe\scomputation\sof\stypes\son\scolumns\sof\sa\sview.
+D 2017-10-03T19:53:12.453
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 764d5ac2bc4d96a9183d8ee3067f19ef4133aa84788d89875e9626f906703171
+F src/select.c fa9fd8750cc55c4367a77fcf1911a770049c947d26d4a3b1e2b9eebf63425f0a
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1dd828088d6981dfebf9f4d650dad8431bece4405650c61f90eb8d8f43289b52
-R ff9a44814ead03fb339e5473f4e89eb3
+P f5c395834c2a776beba6fe172cc4a5e428ce30b9cb4259ef8e440f10455a41e9
+R 3c121170a1dd33a49ba61d24de5be781
 U drh
-Z 7ee7c6afb266c430d526e6cc80b0cc29
+Z 7bba01406d4dc6644f9e7f5f6c664e0a
diff --git a/manifest.uuid b/manifest.uuid
index bbdc43be85..b336d47568 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f5c395834c2a776beba6fe172cc4a5e428ce30b9cb4259ef8e440f10455a41e9
\ No newline at end of file
+772b0db1469c9e1c2728cf65dd070e29c624e75c3a5da72ee1297f15d758dd13
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index f628e56f64..63078a9a2b 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1800,7 +1800,8 @@ void sqlite3SelectAddColumnTypeAndCollation(
     zType = columnType(&sNC, p, 0, 0, 0);
     /* pCol->szEst = ... // Column size est for SELECT tables never used */
     pCol->affinity = sqlite3ExprAffinity(p);
-    if( zType && (m = sqlite3Strlen30(zType))>0 ){
+    if( zType ){
+      m = sqlite3Strlen30(zType);
       n = sqlite3Strlen30(pCol->zName);
       pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2);
       if( pCol->zName ){

From e76acc654f196fdef2d0c1322300be962da36ea8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 4 Oct 2017 02:30:45 +0000
Subject: [PATCH 146/270] Fix a minor typo in the query flattener header
 comment.  No code changes.

FossilOrigin-Name: d050dc605c24bcf60c0c47d13612ad53b871d3d4eff681c0c1b933acf53fb5ee
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b9988d900e..b5978e8b5e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplify\sthe\scomputation\sof\stypes\son\scolumns\sof\sa\sview.
-D 2017-10-03T19:53:12.453
+C Fix\sa\sminor\stypo\sin\sthe\squery\sflattener\sheader\scomment.\s\sNo\scode\schanges.
+D 2017-10-04T02:30:45.805
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c fa9fd8750cc55c4367a77fcf1911a770049c947d26d4a3b1e2b9eebf63425f0a
+F src/select.c 4248b28ec9db68653d3ea5137b7c7f5bad52451486ccf1a2247cd39ca3b9e35c
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f5c395834c2a776beba6fe172cc4a5e428ce30b9cb4259ef8e440f10455a41e9
-R 3c121170a1dd33a49ba61d24de5be781
+P 772b0db1469c9e1c2728cf65dd070e29c624e75c3a5da72ee1297f15d758dd13
+R 5f83cd7ed7c6212a42bd83dd65e953e5
 U drh
-Z 7bba01406d4dc6644f9e7f5f6c664e0a
+Z 4de48cca341b3f710b375c35ea16b291
diff --git a/manifest.uuid b/manifest.uuid
index b336d47568..0509f11b51 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-772b0db1469c9e1c2728cf65dd070e29c624e75c3a5da72ee1297f15d758dd13
\ No newline at end of file
+d050dc605c24bcf60c0c47d13612ad53b871d3d4eff681c0c1b933acf53fb5ee
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 63078a9a2b..0ef316a540 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3355,7 +3355,7 @@ static void substSelect(
 **  (17)  If the subquery is a compound select, then
 **        (17a) all compound operators must be a UNION ALL, and
 **        (17b) no terms within the subquery compound may be aggregate
-**              or DISTINT, and
+**              or DISTINCT, and
 **        (17c) every term within the subquery compound must have a FROM clause
 **        (17d) the outer query may not be
 **              (17d1) aggregate, or

From cdb2f60743dad83e108e4600e9eab6a935c311ec Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 4 Oct 2017 05:59:54 +0000
Subject: [PATCH 147/270] Remove a redundant restriction from the query
 flattener.

FossilOrigin-Name: 66629b2a0997ceedcfb38553f2200466b6c4e352ea00f8a0a7cb67a660c19523
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  | 16 +++++++++++-----
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index b5978e8b5e..37d62ef094 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sminor\stypo\sin\sthe\squery\sflattener\sheader\scomment.\s\sNo\scode\schanges.
-D 2017-10-04T02:30:45.805
+C Remove\sa\sredundant\srestriction\sfrom\sthe\squery\sflattener.
+D 2017-10-04T05:59:54.466
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 4248b28ec9db68653d3ea5137b7c7f5bad52451486ccf1a2247cd39ca3b9e35c
+F src/select.c 8a506e0a27799e96276aea0b5fd1591762905bd572f326d2886cb74f5ceb5f4a
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 772b0db1469c9e1c2728cf65dd070e29c624e75c3a5da72ee1297f15d758dd13
-R 5f83cd7ed7c6212a42bd83dd65e953e5
+P d050dc605c24bcf60c0c47d13612ad53b871d3d4eff681c0c1b933acf53fb5ee
+R 0deb4a35d3602fd5f83a95efc798c5e9
 U drh
-Z 4de48cca341b3f710b375c35ea16b291
+Z 70da60dffddabeaa8e38379103c330a2
diff --git a/manifest.uuid b/manifest.uuid
index 0509f11b51..c73fafb1b1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d050dc605c24bcf60c0c47d13612ad53b871d3d4eff681c0c1b933acf53fb5ee
\ No newline at end of file
+66629b2a0997ceedcfb38553f2200466b6c4e352ea00f8a0a7cb67a660c19523
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 0ef316a540..6b5770e737 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3393,8 +3393,9 @@ static void substSelect(
 **
 **  (22)  The subquery may not be a recursive CTE.
 **
-**  (23)  If the outer query is a recursive CTE, then the sub-query may not be
-**        a compound query. This restriction is because transforming the
+**  (**)  Subsumed into restriction (17d3).  Was: If the outer query is
+**        a recursive CTE, then the sub-query may not be a compound query.
+**        This restriction is because transforming the
 **        parent to a compound query confuses the code that handles
 **        recursive queries in multiSelect().
 **
@@ -3475,9 +3476,6 @@ static int flattenSubquery(
   if( pSub->selFlags & (SF_Recursive) ){
     return 0; /* Restrictions (22) */
   }
-  if( (p->selFlags & SF_Recursive) && pSub->pPrior ){
-    return 0; /* Restriction (23) */
-  }
 
   /*
   ** If the subquery is the right operand of a LEFT JOIN, then the
@@ -3551,6 +3549,14 @@ static int flattenSubquery(
     }
   }
 
+  /* Ex-restriction (23):
+  ** The only way that the recursive part of a CTE can contain a compound
+  ** subquery is for the subquery to be one term of a join.  But if the
+  ** subquery is a join, then the flattening has already been stopped by
+  ** restriction (17d3)
+  */
+  assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 );
+
   /***** If we reach this point, flattening is permitted. *****/
   SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n",
                    pSub->zSelName, pSub, iFrom));

From 61b513e9e310810b5f5405db45b22fb134060a1d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 4 Oct 2017 10:39:28 +0000
Subject: [PATCH 148/270] Add tests to verify that the busy-handler is invoked
 correctly when processing "PRAGMA optimize" and ANALYZE commands.

FossilOrigin-Name: fb83c3d8df250cb701fbe775b48ab93f5674496f68c57e04f50668c43c2de328
---
 manifest       | 14 ++++-----
 manifest.uuid  |  2 +-
 test/busy.test | 82 ++++++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 88 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 37d62ef094..faadfc4de1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\sredundant\srestriction\sfrom\sthe\squery\sflattener.
-D 2017-10-04T05:59:54.466
+C Add\stests\sto\sverify\sthat\sthe\sbusy-handler\sis\sinvoked\scorrectly\swhen\sprocessing\s"PRAGMA\soptimize"\sand\sANALYZE\scommands.
+D 2017-10-04T10:39:28.853
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -633,7 +633,7 @@ F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b
 F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f
 F test/btree02.test fe69453d474d8154d19b904157ff1db4812fed99
 F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3
-F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0
+F test/busy.test bf7f099161793dd2e50b0e657c68840ac6ad19d4af2f34ba6702ace67a00040d
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d050dc605c24bcf60c0c47d13612ad53b871d3d4eff681c0c1b933acf53fb5ee
-R 0deb4a35d3602fd5f83a95efc798c5e9
-U drh
-Z 70da60dffddabeaa8e38379103c330a2
+P 66629b2a0997ceedcfb38553f2200466b6c4e352ea00f8a0a7cb67a660c19523
+R 9a9c201e8296e0260f25603d504d4def
+U dan
+Z 0d6df17481534a9055059794f33541b5
diff --git a/manifest.uuid b/manifest.uuid
index c73fafb1b1..97fd34fece 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-66629b2a0997ceedcfb38553f2200466b6c4e352ea00f8a0a7cb67a660c19523
\ No newline at end of file
+fb83c3d8df250cb701fbe775b48ab93f5674496f68c57e04f50668c43c2de328
\ No newline at end of file
diff --git a/test/busy.test b/test/busy.test
index 585f764547..e2f13cbebe 100644
--- a/test/busy.test
+++ b/test/busy.test
@@ -10,11 +10,11 @@
 #***********************************************************************
 # This file test the busy handler
 #
-# $Id: busy.test,v 1.3 2008/03/15 02:09:22 drh Exp $
 
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix busy
 
 do_test busy-1.1 {
   sqlite3 db2 test.db
@@ -55,7 +55,85 @@ do_test busy-2.2 {
   set busyargs
 } {0 1 2 3}
 
-
 db2 close
 
+#-------------------------------------------------------------------------
+# Test that the busy-handler is invoked correctly for "PRAGMA optimize"
+# and ANALYZE commnds.
+ifcapable pragma&&analyze {
+
+reset_db
+
+do_execsql_test 2.1 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  CREATE TABLE t3(z);
+
+  CREATE INDEX i1 ON t1(x);
+  CREATE INDEX i2 ON t2(y);
+
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t2 VALUES(1);
+  ANALYZE;
+
+  SELECT * FROM t1 WHERE x=1;
+  SELECT * FROM t2 WHERE y=1;
+} {1 1}
+
+do_test 2.2 {
+  sqlite3 db2 test.db
+  execsql { BEGIN EXCLUSIVE } db2
+  catchsql { PRAGMA optimize }
+} {1 {database is locked}}
+
+proc busy_handler {n} {
+  if {$n>1000} { execsql { COMMIT } db2 }
+  return 0
+}
+db busy busy_handler
+
+do_test 2.3 {
+  catchsql { PRAGMA optimize }
+} {0 {}}
+
+do_test 2.4 {
+  execsql {
+    BEGIN;
+    SELECT count(*) FROM sqlite_master;
+  } db2
+} {6}
+
+proc busy_handler {n} { return 1 }
+do_test 2.5 {
+  catchsql { PRAGMA optimize }
+} {0 {}}
+
+do_test 2.6 {
+  execsql { COMMIT } db2
+  execsql {
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t1 SELECT i FROM s;
+  }
+  execsql {
+    BEGIN;
+    SELECT count(*) FROM sqlite_master;
+  } db2
+} {6}
+
+do_test 2.7 {
+  catchsql { PRAGMA optimize }
+} {1 {database is locked}}
+
+proc busy_handler {n} {
+  if {$n>1000} { execsql { COMMIT } db2 }
+  return 0
+}
+do_test 2.8 {
+  catchsql { PRAGMA optimize }
+} {0 {}}
+
+}
+
 finish_test

From 7c58fea39f9f3c19f4ad0c13e0e93c0943ef84c7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 4 Oct 2017 12:06:31 +0000
Subject: [PATCH 149/270] Turn restriction 20 on the query flattener into an
 assert since the situation restricted can no longer occur because of the more
 aggressive use of co-routines.

FossilOrigin-Name: 4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/select.c  | 23 +++++++++++++++--------
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index faadfc4de1..23a196d039 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sto\sverify\sthat\sthe\sbusy-handler\sis\sinvoked\scorrectly\swhen\sprocessing\s"PRAGMA\soptimize"\sand\sANALYZE\scommands.
-D 2017-10-04T10:39:28.853
+C Turn\srestriction\s20\son\sthe\squery\sflattener\sinto\san\sassert\ssince\sthe\ssituation\nrestricted\scan\sno\slonger\soccur\sbecause\sof\sthe\smore\saggressive\suse\sof\nco-routines.
+D 2017-10-04T12:06:31.705
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 8a506e0a27799e96276aea0b5fd1591762905bd572f326d2886cb74f5ceb5f4a
+F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
 F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 66629b2a0997ceedcfb38553f2200466b6c4e352ea00f8a0a7cb67a660c19523
-R 9a9c201e8296e0260f25603d504d4def
-U dan
-Z 0d6df17481534a9055059794f33541b5
+P fb83c3d8df250cb701fbe775b48ab93f5674496f68c57e04f50668c43c2de328
+R f3d3274b176630163255462be6322523
+U drh
+Z 54cf8654d3e3811fdefd23ba6c7d6b42
diff --git a/manifest.uuid b/manifest.uuid
index 97fd34fece..d079033e67 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fb83c3d8df250cb701fbe775b48ab93f5674496f68c57e04f50668c43c2de328
\ No newline at end of file
+4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 6b5770e737..6f524fa2e1 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3382,11 +3382,12 @@ static void substSelect(
 **  (19)  If the subquery uses LIMIT then the outer query may not
 **        have a WHERE clause.
 **
-**  (20)  If the sub-query is a compound select, then it must not use
-**        an ORDER BY clause.  Ticket #3773.  We could relax this constraint
-**        somewhat by saying that the terms of the ORDER BY clause must
-**        appear as unmodified result columns in the outer query.  But we
-**        have other optimizations in mind to deal with that case.
+**  (**)  Subsumed into (17d3).  Was: If the sub-query is a compound select,
+**        then it must not use an ORDER BY clause - Ticket #3773.  Because
+**        of (17d3), then only way to have a compound subquery is if it is
+**        the only term in the FROM clause of the outer query.  But if the
+**        only term in the FROM clause has an ORDER BY, then it will be
+**        implemented as a co-routine and the flattener will never be called.
 **
 **  (21)  If the subquery uses LIMIT then the outer query may not be
 **        DISTINCT.  (See ticket [752e1646fc]).
@@ -3520,9 +3521,6 @@ static int flattenSubquery(
   ** queries.
   */
   if( pSub->pPrior ){
-    if( pSub->pOrderBy ){
-      return 0;  /* Restriction (20) */
-    }
     if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
       return 0; /* (17d1), (17d2), or (17d3) */
     }
@@ -3557,6 +3555,15 @@ static int flattenSubquery(
   */
   assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 );
 
+  /* Ex-restriction (20):
+  ** A compound subquery must be the only term in the FROM clause of the
+  ** outer query by restriction (17d3).  But if that term also has an
+  ** ORDER BY clause, then the subquery will be implemented by co-routine
+  ** and so the flattener will never be invoked.  Hence, it is not possible
+  ** for the subquery to be a compound and have an ORDER BY clause.
+  */
+  assert( pSub->pPrior==0 || pSub->pOrderBy==0 );
+
   /***** If we reach this point, flattening is permitted. *****/
   SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n",
                    pSub->zSelName, pSub, iFrom));

From 4c16760c373e24e2b6116086f4437ee6ee4c7861 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 4 Oct 2017 12:08:35 +0000
Subject: [PATCH 150/270] Amend recent changes to busy.test to avoid failing
 with SQLITE_ENABLE_STAT4 builds.

FossilOrigin-Name: 9ff4944194c56b38843bc626dc51604db8e1ca8a0fd221e2d6f78c840d57c341
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 test/busy.test | 18 +++++++++---------
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 23a196d039..ae0689f471 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Turn\srestriction\s20\son\sthe\squery\sflattener\sinto\san\sassert\ssince\sthe\ssituation\nrestricted\scan\sno\slonger\soccur\sbecause\sof\sthe\smore\saggressive\suse\sof\nco-routines.
-D 2017-10-04T12:06:31.705
+C Amend\srecent\schanges\sto\sbusy.test\sto\savoid\sfailing\swith\sSQLITE_ENABLE_STAT4\nbuilds.
+D 2017-10-04T12:08:35.867
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -633,7 +633,7 @@ F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b
 F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f
 F test/btree02.test fe69453d474d8154d19b904157ff1db4812fed99
 F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3
-F test/busy.test bf7f099161793dd2e50b0e657c68840ac6ad19d4af2f34ba6702ace67a00040d
+F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb83c3d8df250cb701fbe775b48ab93f5674496f68c57e04f50668c43c2de328
-R f3d3274b176630163255462be6322523
-U drh
-Z 54cf8654d3e3811fdefd23ba6c7d6b42
+P 4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
+R 4c5ee8ddebc2e3331db784299bd3d5dd
+U dan
+Z 592fc1b29722b14ffc2bc21290e229f0
diff --git a/manifest.uuid b/manifest.uuid
index d079033e67..c47875e678 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
\ No newline at end of file
+9ff4944194c56b38843bc626dc51604db8e1ca8a0fd221e2d6f78c840d57c341
\ No newline at end of file
diff --git a/test/busy.test b/test/busy.test
index e2f13cbebe..be0515b013 100644
--- a/test/busy.test
+++ b/test/busy.test
@@ -60,11 +60,11 @@ db2 close
 #-------------------------------------------------------------------------
 # Test that the busy-handler is invoked correctly for "PRAGMA optimize"
 # and ANALYZE commnds.
-ifcapable pragma&&analyze {
+ifcapable pragma&&analyze&&!stat4 {
 
 reset_db
 
-do_execsql_test 2.1 {
+do_execsql_test 3.1 {
   CREATE TABLE t1(x);
   CREATE TABLE t2(y);
   CREATE TABLE t3(z);
@@ -80,7 +80,7 @@ do_execsql_test 2.1 {
   SELECT * FROM t2 WHERE y=1;
 } {1 1}
 
-do_test 2.2 {
+do_test 3.2 {
   sqlite3 db2 test.db
   execsql { BEGIN EXCLUSIVE } db2
   catchsql { PRAGMA optimize }
@@ -92,11 +92,11 @@ proc busy_handler {n} {
 }
 db busy busy_handler
 
-do_test 2.3 {
+do_test 3.3 {
   catchsql { PRAGMA optimize }
 } {0 {}}
 
-do_test 2.4 {
+do_test 3.4 {
   execsql {
     BEGIN;
     SELECT count(*) FROM sqlite_master;
@@ -104,11 +104,11 @@ do_test 2.4 {
 } {6}
 
 proc busy_handler {n} { return 1 }
-do_test 2.5 {
+do_test 3.5 {
   catchsql { PRAGMA optimize }
 } {0 {}}
 
-do_test 2.6 {
+do_test 3.6 {
   execsql { COMMIT } db2
   execsql {
     WITH s(i) AS (
@@ -122,7 +122,7 @@ do_test 2.6 {
   } db2
 } {6}
 
-do_test 2.7 {
+do_test 3.7 {
   catchsql { PRAGMA optimize }
 } {1 {database is locked}}
 
@@ -130,7 +130,7 @@ proc busy_handler {n} {
   if {$n>1000} { execsql { COMMIT } db2 }
   return 0
 }
-do_test 2.8 {
+do_test 3.8 {
   catchsql { PRAGMA optimize }
 } {0 {}}
 

From f3b2c7aa81b43897ebb9131a48ab367c9e616528 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 4 Oct 2017 14:13:29 +0000
Subject: [PATCH 151/270] New test cases for ticket [b899b6042f97f5] derived
 from a bug report on the mailing list from Wout Mertens.

FossilOrigin-Name: ef94ea061d2ec8ee0243e97dfcfc7a7c75dd22006e61a7195b780722bbc925db
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 test/whereF.test | 19 ++++++++++++++++++-
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index ae0689f471..072511d38f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Amend\srecent\schanges\sto\sbusy.test\sto\savoid\sfailing\swith\sSQLITE_ENABLE_STAT4\nbuilds.
-D 2017-10-04T12:08:35.867
+C New\stest\scases\sfor\sticket\s[b899b6042f97f5]\sderived\sfrom\sa\sbug\sreport\son\nthe\smailing\slist\sfrom\sWout\sMertens.
+D 2017-10-04T14:13:29.708
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1534,7 +1534,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test 7c426e0bf303407910c21f79d3a6742f7b33c9b27b2dddd230dfc8c2508981a6
+F test/whereF.test a44c7d73c22e2532a5c2dc7377fca64825f86a8969b113a1bdcffacbcbc14d39
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
-R 4c5ee8ddebc2e3331db784299bd3d5dd
-U dan
-Z 592fc1b29722b14ffc2bc21290e229f0
+P 9ff4944194c56b38843bc626dc51604db8e1ca8a0fd221e2d6f78c840d57c341
+R 799bca1879769404f931f1637b51cab6
+U drh
+Z bd1c8b775635a1239ba29ee4684e681f
diff --git a/manifest.uuid b/manifest.uuid
index c47875e678..d1bec2640f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9ff4944194c56b38843bc626dc51604db8e1ca8a0fd221e2d6f78c840d57c341
\ No newline at end of file
+ef94ea061d2ec8ee0243e97dfcfc7a7c75dd22006e61a7195b780722bbc925db
\ No newline at end of file
diff --git a/test/whereF.test b/test/whereF.test
index a2361cfd16..0888f25789 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -195,7 +195,24 @@ ifcapable json1 {
     SELECT * FROM t6
      WHERE (EXISTS (SELECT 1 FROM json_each(t6.c) AS x WHERE x.value=1));
   } {1 {} {{"a":1,"b":[3,4,5],"c":{"x":4.5,"y":7.8}}}}
-    
+
+  # Another test case derived from a posting by Wout Mertens on the
+  # sqlite-users mailing list on 2017-10-04.
+  do_execsql_test 6.3 {
+    DROP TABLE IF EXISTS t;
+    CREATE TABLE t(json JSON);
+    SELECT * FROM t
+     WHERE(EXISTS(SELECT 1 FROM json_each(t.json,"$.foo") j
+                   WHERE j.value = 'meep'));
+  } {}
+  do_execsql_test 6.4 {
+    INSERT INTO t VALUES('{"xyzzy":null}');
+    INSERT INTO t VALUES('{"foo":"meep","other":12345}');
+    INSERT INTO t VALUES('{"foo":"bingo","alt":5.25}');
+    SELECT * FROM t
+     WHERE(EXISTS(SELECT 1 FROM json_each(t.json,"$.foo") j
+                   WHERE j.value = 'meep'));
+  } {{{"foo":"meep","other":12345}}}
 }
 
 finish_test

From 3dfbe9b3fc25614a55e8b311436625c99c757d75 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 4 Oct 2017 16:35:01 +0000
Subject: [PATCH 152/270] Fix typos in session extension header comments.  No
 changes to code.

FossilOrigin-Name: 182ec9d6f6c44938de0a2cd542e259d082deeeabc12a1815fff35e3ed1e524ef
---
 ext/session/sqlite3session.h | 16 ++++++++--------
 manifest                     | 14 +++++++-------
 manifest.uuid                |  2 +-
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index 0882cf34cf..b22df129cb 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -375,8 +375,8 @@ int sqlite3session_diff(
 */
 int sqlite3session_patchset(
   sqlite3_session *pSession,      /* Session object */
-  int *pnPatchset,                /* OUT: Size of buffer at *ppChangeset */
-  void **ppPatchset               /* OUT: Buffer containing changeset */
+  int *pnPatchset,                /* OUT: Size of buffer at *ppPatchset */
+  void **ppPatchset               /* OUT: Buffer containing patchset */
 );
 
 /*
@@ -1143,12 +1143,12 @@ int sqlite3changeset_apply(
 **
 ** <table border=1 style="margin-left:8ex;margin-right:8ex">
 **   <tr><th>Streaming function<th>Non-streaming equivalent</th>
-**   <tr><td>sqlite3changeset_apply_str<td>[sqlite3changeset_apply] 
-**   <tr><td>sqlite3changeset_concat_str<td>[sqlite3changeset_concat] 
-**   <tr><td>sqlite3changeset_invert_str<td>[sqlite3changeset_invert] 
-**   <tr><td>sqlite3changeset_start_str<td>[sqlite3changeset_start] 
-**   <tr><td>sqlite3session_changeset_str<td>[sqlite3session_changeset] 
-**   <tr><td>sqlite3session_patchset_str<td>[sqlite3session_patchset] 
+**   <tr><td>sqlite3changeset_apply_strm<td>[sqlite3changeset_apply] 
+**   <tr><td>sqlite3changeset_concat_strm<td>[sqlite3changeset_concat] 
+**   <tr><td>sqlite3changeset_invert_strm<td>[sqlite3changeset_invert] 
+**   <tr><td>sqlite3changeset_start_strm<td>[sqlite3changeset_start] 
+**   <tr><td>sqlite3session_changeset_strm<td>[sqlite3session_changeset] 
+**   <tr><td>sqlite3session_patchset_strm<td>[sqlite3session_patchset] 
 ** </table>
 **
 ** Non-streaming functions that accept changesets (or patchsets) as input
diff --git a/manifest b/manifest
index 072511d38f..204d281625 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\stest\scases\sfor\sticket\s[b899b6042f97f5]\sderived\sfrom\sa\sbug\sreport\son\nthe\smailing\slist\sfrom\sWout\sMertens.
-D 2017-10-04T14:13:29.708
+C Fix\stypos\sin\ssession\sextension\sheader\scomments.\s\sNo\schanges\sto\scode.
+D 2017-10-04T16:35:01.122
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -374,7 +374,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c cc127222a9ea6f4eaa31281aa9da924f5244f6099be0ee526c950684fb3513a6
-F ext/session/sqlite3session.h d4db650adfcc7a4360e9f12a09c2d117b1db6b53
+F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9ff4944194c56b38843bc626dc51604db8e1ca8a0fd221e2d6f78c840d57c341
-R 799bca1879769404f931f1637b51cab6
-U drh
-Z bd1c8b775635a1239ba29ee4684e681f
+P ef94ea061d2ec8ee0243e97dfcfc7a7c75dd22006e61a7195b780722bbc925db
+R c7a044ee466229738035e85eab787e7b
+U mistachkin
+Z 9b266a68687cdc8bad7e7c02276ace92
diff --git a/manifest.uuid b/manifest.uuid
index d1bec2640f..2b8c16c9a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ef94ea061d2ec8ee0243e97dfcfc7a7c75dd22006e61a7195b780722bbc925db
\ No newline at end of file
+182ec9d6f6c44938de0a2cd542e259d082deeeabc12a1815fff35e3ed1e524ef
\ No newline at end of file

From 7b7c956f09dd368b6ff7c955ce7b141144d9d1d7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 4 Oct 2017 17:05:04 +0000
Subject: [PATCH 153/270] Update the speedtest1 program so that it prints the
 output of "PRAGMA compile_options" if the -stats option is specified.

FossilOrigin-Name: 7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 test/speedtest1.c |  9 +++++++++
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 204d281625..c81f7276e9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypos\sin\ssession\sextension\sheader\scomments.\s\sNo\schanges\sto\scode.
-D 2017-10-04T16:35:01.122
+C Update\sthe\sspeedtest1\sprogram\sso\sthat\sit\sprints\sthe\soutput\sof\s"PRAGMA\ncompile_options"\sif\sthe\s-stats\soption\sis\sspecified.
+D 2017-10-04T17:05:04.541
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1223,7 +1223,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 7a6ec22e87f78ef98d523667593f5d818f6be4a1bcf5fe70d933aece058f32df
+F test/speedtest1.c e44c5fccddcfe916c3bf7fe2f87dcc4b4fd66a0d923eb83515f311212670f267
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ef94ea061d2ec8ee0243e97dfcfc7a7c75dd22006e61a7195b780722bbc925db
-R c7a044ee466229738035e85eab787e7b
-U mistachkin
-Z 9b266a68687cdc8bad7e7c02276ace92
+P 182ec9d6f6c44938de0a2cd542e259d082deeeabc12a1815fff35e3ed1e524ef
+R c6410b6021916e7e52f7510d770d1c78
+U dan
+Z 9d814cce16c052d0d52ffa24753c3c91
diff --git a/manifest.uuid b/manifest.uuid
index 2b8c16c9a7..6a52093cc9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-182ec9d6f6c44938de0a2cd542e259d082deeeabc12a1815fff35e3ed1e524ef
\ No newline at end of file
+7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f
\ No newline at end of file
diff --git a/test/speedtest1.c b/test/speedtest1.c
index e374c22c1e..b92801a208 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -1635,6 +1635,11 @@ static void displayLinuxIoStats(FILE *out){
 #  define sqlite3_sourceid(X) "(before 3.6.18)"
 #endif
 
+static int xCompileOptions(void *pCtx, int nVal, char **azVal, char **azCol){
+  printf("-- Compile option: %s\n", azVal[0]);
+  return SQLITE_OK;
+}
+
 int main(int argc, char **argv){
   int doAutovac = 0;            /* True for --autovacuum */
   int cacheSize = 0;            /* Desired cache size.  0 means default */
@@ -1881,6 +1886,10 @@ int main(int argc, char **argv){
   }
   speedtest1_final();
 
+  if( showStats ){
+    sqlite3_exec(g.db, "PRAGMA compile_options", xCompileOptions, 0, 0);
+  }
+
   /* Database connection statistics printed after both prepared statements
   ** have been finalized */
 #if SQLITE_VERSION_NUMBER>=3007009

From c6aa38159be44e4ba837b32d9bdf493f488ac03e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 4 Oct 2017 18:26:44 +0000
Subject: [PATCH 154/270] Updates to requirements marks.

FossilOrigin-Name: 40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422
---
 manifest          | 20 ++++++++++----------
 manifest.uuid     |  2 +-
 src/btree.c       |  2 --
 src/main.c        |  6 ++++--
 test/attach2.test |  3 +--
 test/e_uri.test   |  4 ++--
 6 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index c81f7276e9..1ea0bfb8d7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sspeedtest1\sprogram\sso\sthat\sit\sprints\sthe\soutput\sof\s"PRAGMA\ncompile_options"\sif\sthe\s-stats\soption\sis\sspecified.
-D 2017-10-04T17:05:04.541
+C Updates\sto\srequirements\smarks.
+D 2017-10-04T18:26:44.610
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -401,7 +401,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 221bc1b836f0c386676999a7c62c8dc60455e255fab37df97eca2aa619b92f2a
+F src/btree.c cc88a7fca7287dfc004921bb5e2764893dfe4f6dd33be3570126b3fc37932600
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
@@ -423,7 +423,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 556a0942bdab9cd33f75aaec4adf54cc008898cb73b9657337be5960c7bcac8f
+F src/main.c a4bdadaaa827e7380cba4de878ed7947dab5aeb84f617118ba6a0422cd745b4b
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -580,7 +580,7 @@ F test/async5.test 383ab533fdb9f7ad228cc99ee66e1acb34cc0dc0
 F test/atof1.test ff0b0156fd705b67c506e1f2bfe9e26102bea9bd
 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061
 F test/attach.test f4b8918ba2f3e88e6883b8452340545f10a1388af808343c37fc5c577be8281c
-F test/attach2.test 567047a7607aae8ebb3794642ebb168abe66b4af366fcd0cf7f616a1495cd43f
+F test/attach2.test 256bd240da1835fb8408dd59fb7ef71f8358c7a756c46662434d11d07ba3a0ce
 F test/attach3.test c59d92791070c59272e00183b7353eeb94915976
 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c
 F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0
@@ -742,7 +742,7 @@ F test/e_select.test 16651bb681e83a1a2875ff4a595ed2b4b4dee375
 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f
 F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10
 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528
-F test/e_uri.test 25385396082b67fd02ae0038b95a3b3575fe0519
+F test/e_uri.test 47eeb2960e74613f0f8722b2f13aef08fde69daa16e5380ac93df84dac8b1f72
 F test/e_vacuum.test 1b8b4772d05374aa1b8958669138bbb4213ee26a
 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625
 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 182ec9d6f6c44938de0a2cd542e259d082deeeabc12a1815fff35e3ed1e524ef
-R c6410b6021916e7e52f7510d770d1c78
-U dan
-Z 9d814cce16c052d0d52ffa24753c3c91
+P 7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f
+R 61b14913c42f4f709d3e3a3845316f97
+U drh
+Z 37903880ad327c432f24949839963d82
diff --git a/manifest.uuid b/manifest.uuid
index 6a52093cc9..c2c8cc6040 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f
\ No newline at end of file
+40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 7c468f35a5..cadd711e45 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -7301,8 +7301,6 @@ static int balance_nonroot(
      + nMaxCells*sizeof(u16)                       /* b.szCell */
      + pBt->pageSize;                              /* aSpace1 */
 
-  /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer
-  ** that is more than 6 times the database page size. */
   assert( szScratch<=6*(int)pBt->pageSize );
   b.apCell = sqlite3StackAllocRaw(0, szScratch );
   if( b.apCell==0 ){
diff --git a/src/main.c b/src/main.c
index 082b96f960..3d7609ce5b 100644
--- a/src/main.c
+++ b/src/main.c
@@ -47,9 +47,11 @@ const char sqlite3_version[] = SQLITE_VERSION;
 */
 const char *sqlite3_libversion(void){ return sqlite3_version; }
 
-/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a
+/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a
 ** pointer to a string constant whose value is the same as the
-** SQLITE_SOURCE_ID C preprocessor macro. 
+** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using
+** an edited copy of the amalgamation, then the last four characters of
+** the hash might be different from SQLITE_SOURCE_ID.
 */
 const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }
 
diff --git a/test/attach2.test b/test/attach2.test
index 741f959372..eb87b60f63 100644
--- a/test/attach2.test
+++ b/test/attach2.test
@@ -378,8 +378,7 @@ do_test attach2-6.2 {
   }
 } {0 {}}
 
-# EVIDENCE-OF: R-59740-55581 This statement will fail if SQLite is in
-# the middle of a transaction.
+# As of version 3.21.0: it is ok to DETACH from within a transaction
 #
 do_test attach2-6.3 {
   catchsql {
diff --git a/test/e_uri.test b/test/e_uri.test
index 95b53f1d6b..dbcc6f3e0b 100644
--- a/test/e_uri.test
+++ b/test/e_uri.test
@@ -50,8 +50,8 @@ proc open_uri_error {uri} {
 # and the filename argument begins with "file:", then the filename is
 # interpreted as a URI.
 #
-# EVIDENCE-OF: R-24124-56960 URI filename interpretation is enabled if
-# the SQLITE_OPEN_URI flag is set in the fourth argument to
+# EVIDENCE-OF: R-27632-24205 URI filename interpretation is enabled if
+# the SQLITE_OPEN_URI flag is set in the third argument to
 # sqlite3_open_v2(), or if it has been enabled globally using the
 # SQLITE_CONFIG_URI option with the sqlite3_config() method or by the
 # SQLITE_USE_URI compile-time option.

From 5aa1e7cd307814427dbc05d8c4a73ce5e3528e75 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 5 Oct 2017 11:29:37 +0000
Subject: [PATCH 155/270] Disable the delta checksum verification in RBU by
 default.  Reenable it using -DRBU_ENABLE_DELTA_CKSUM, if desired.

FossilOrigin-Name: d22c99b6ba997179ef3ccd341d9c36d5213d699305d15942d82587a0bfd16f9d
---
 ext/rbu/sqlite3rbu.c | 13 +++++++++++--
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c
index fbcfab9104..0289ef7ef5 100644
--- a/ext/rbu/sqlite3rbu.c
+++ b/ext/rbu/sqlite3rbu.c
@@ -96,6 +96,13 @@
 /* Maximum number of prepared UPDATE statements held by this module */
 #define SQLITE_RBU_UPDATE_CACHESIZE 16
 
+/* Delta checksums disabled by default.  Compile with -DRBU_ENABLE_DELTA_CKSUM
+** to enable checksum verification.
+*/
+#ifndef RBU_ENABLE_DELTA_CKSUM
+# define RBU_ENABLE_DELTA_CKSUM 0
+#endif
+
 /*
 ** Swap two objects of type TYPE.
 */
@@ -470,6 +477,7 @@ static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
   return v;
 }
 
+#if RBU_ENABLE_DELTA_CKSUM
 /*
 ** Compute a 32-bit checksum on the N-byte buffer.  Return the result.
 */
@@ -504,6 +512,7 @@ static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
   }
   return sum3;
 }
+#endif
 
 /*
 ** Apply a delta.
@@ -534,7 +543,7 @@ static int rbuDeltaApply(
 ){
   unsigned int limit;
   unsigned int total = 0;
-#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
+#if RBU_ENABLE_DELTA_CKSUM
   char *zOrigOut = zOut;
 #endif
 
@@ -589,7 +598,7 @@ static int rbuDeltaApply(
       case ';': {
         zDelta++; lenDelta--;
         zOut[0] = 0;
-#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
+#if RBU_ENABLE_DELTA_CKSUM
         if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
           /* ERROR:  bad checksum */
           return -1;
diff --git a/manifest b/manifest
index 1ea0bfb8d7..f6f9c47a6f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\srequirements\smarks.
-D 2017-10-04T18:26:44.610
+C Disable\sthe\sdelta\schecksum\sverification\sin\sRBU\sby\sdefault.\s\sReenable\sit\susing\n-DRBU_ENABLE_DELTA_CKSUM,\sif\sdesired.
+D 2017-10-05T11:29:37.874
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -322,7 +322,7 @@ F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/rbutemplimit.test cd553a9288d515d0b5f87d277e76fd18c4aa740b761e7880fab11ce986ea18d1
 F ext/rbu/rbuvacuum.test ff357e9b556ca7ad4673da0ff7f244def919ff858e0f9f350d3e30fdd83a62a8
 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa
-F ext/rbu/sqlite3rbu.c a1a303de8b90f987ef63bf9cef57f5d7dd7983a9e8aed3775a759d87ad57075d
+F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078da04feb
 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7c69f8f1089c3e3843fbf7ec37a897c849a3df822a4ce3b4fcde586adf991a3f
-R 61b14913c42f4f709d3e3a3845316f97
+P 40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422
+R 56c4429917e25e309d2b170a348071dc
 U drh
-Z 37903880ad327c432f24949839963d82
+Z 02d9f9a2aecdfad7c3d7a847ab2e4ae9
diff --git a/manifest.uuid b/manifest.uuid
index c2c8cc6040..62bdb52227 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422
\ No newline at end of file
+d22c99b6ba997179ef3ccd341d9c36d5213d699305d15942d82587a0bfd16f9d
\ No newline at end of file

From c3cbd678a1480c53dec3b8f986747c6686b4dee9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 5 Oct 2017 19:12:10 +0000
Subject: [PATCH 156/270] Fix the command-line shell so that the ".schema
 --indent" command does a better job of dealing with \r\n in the middle of a
 CREATE statement in the schema.

FossilOrigin-Name: 4258fb578a6e75590c7314fe511ca8dc6659e5f2d3a54d8f60ff705fe51ccc92
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c    |  1 +
 src/shell.c.in |  1 +
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index f6f9c47a6f..c8dcd3d77f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sthe\sdelta\schecksum\sverification\sin\sRBU\sby\sdefault.\s\sReenable\sit\susing\n-DRBU_ENABLE_DELTA_CKSUM,\sif\sdesired.
-D 2017-10-05T11:29:37.874
+C Fix\sthe\scommand-line\sshell\sso\sthat\sthe\s".schema\s--indent"\scommand\sdoes\sa\sbetter\njob\sof\sdealing\swith\s\\r\\n\sin\sthe\smiddle\sof\sa\sCREATE\sstatement\sin\sthe\sschema.
+D 2017-10-05T19:12:10.083
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -459,8 +459,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab
-F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee
+F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe
+F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 40964a4ef7565ea0ddf452f48cb22373d068528e07d40eefc008f2231c969422
-R 56c4429917e25e309d2b170a348071dc
+P d22c99b6ba997179ef3ccd341d9c36d5213d699305d15942d82587a0bfd16f9d
+R 275813642399adb31ada00e2bf9958bd
 U drh
-Z 02d9f9a2aecdfad7c3d7a847ab2e4ae9
+Z f0605b7334559ebfe51c1e99fb005c13
diff --git a/manifest.uuid b/manifest.uuid
index 62bdb52227..bfe1bbba34 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d22c99b6ba997179ef3ccd341d9c36d5213d699305d15942d82587a0bfd16f9d
\ No newline at end of file
+4258fb578a6e75590c7314fe511ca8dc6659e5f2d3a54d8f60ff705fe51ccc92
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index 6a19e3610a..f7be941570 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -2773,6 +2773,7 @@ static int shell_callback(
       for(i=0; IsSpace(z[i]); i++){}
       for(; (c = z[i])!=0; i++){
         if( IsSpace(c) ){
+          if( z[j-1]=='\r' ) z[j-1] = '\n';
           if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue;
         }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){
           j--;
diff --git a/src/shell.c.in b/src/shell.c.in
index 82680d484c..db4f2e1128 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1413,6 +1413,7 @@ static int shell_callback(
       for(i=0; IsSpace(z[i]); i++){}
       for(; (c = z[i])!=0; i++){
         if( IsSpace(c) ){
+          if( z[j-1]=='\r' ) z[j-1] = '\n';
           if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue;
         }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){
           j--;

From 5f54e2b5d3fa522dd66666d28843e74e80788875 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 5 Oct 2017 20:02:49 +0000
Subject: [PATCH 157/270] Fix a problem building with SQLITE_OMIT_WAL defined.

FossilOrigin-Name: 373b0ace480aa303bbf512ea8806a17f6186b16d6316a7b724499bf94b3974d4
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/pager.c   |  6 +++++-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index c8dcd3d77f..94c993303c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\scommand-line\sshell\sso\sthat\sthe\s".schema\s--indent"\scommand\sdoes\sa\sbetter\njob\sof\sdealing\swith\s\\r\\n\sin\sthe\smiddle\sof\sa\sCREATE\sstatement\sin\sthe\sschema.
-D 2017-10-05T19:12:10.083
+C Fix\sa\sproblem\sbuilding\swith\sSQLITE_OMIT_WAL\sdefined.
+D 2017-10-05T20:02:49.526
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -445,7 +445,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 2aa56a99bb13128d9102e84c7a9f835e546cbb58f0861d481bc3db32973b1628
+F src/pager.c 62f88892d3a2c68cff6e8f96c81c5dfe5178eace887880c36364aabe4d8d6422
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d22c99b6ba997179ef3ccd341d9c36d5213d699305d15942d82587a0bfd16f9d
-R 275813642399adb31ada00e2bf9958bd
-U drh
-Z f0605b7334559ebfe51c1e99fb005c13
+P 4258fb578a6e75590c7314fe511ca8dc6659e5f2d3a54d8f60ff705fe51ccc92
+R fba1f72a529b91d37df6bb92e25ac897
+U dan
+Z 4d1d6f0cdfff4b56d945b83bdce453fb
diff --git a/manifest.uuid b/manifest.uuid
index bfe1bbba34..d29e532767 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4258fb578a6e75590c7314fe511ca8dc6659e5f2d3a54d8f60ff705fe51ccc92
\ No newline at end of file
+373b0ace480aa303bbf512ea8806a17f6186b16d6316a7b724499bf94b3974d4
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index a43614cdb4..2ddca9a5f7 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -3012,6 +3012,8 @@ end_playback:
 static int readDbPage(PgHdr *pPg){
   Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
   int rc = SQLITE_OK;          /* Return code */
+
+#ifndef SQLITE_OMIT_WAL
   u32 iFrame = 0;              /* Frame of WAL containing pgno */
 
   assert( pPager->eState>=PAGER_READER && !MEMDB );
@@ -3023,7 +3025,9 @@ static int readDbPage(PgHdr *pPg){
   }
   if( iFrame ){
     rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData);
-  }else{
+  }else
+#endif
+  {
     i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize;
     rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset);
     if( rc==SQLITE_IOERR_SHORT_READ ){

From 2ed5737aca813536d27f569dd6ae42425df4e0b3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 5 Oct 2017 20:57:38 +0000
Subject: [PATCH 158/270] Fix compiler warnings that come up with
 SQLITE_OMIT_WAL.

FossilOrigin-Name: 8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135
---
 manifest      | 18 +++++++++---------
 manifest.uuid |  2 +-
 src/btree.c   |  3 ++-
 src/os.c      |  2 ++
 src/os.h      |  2 ++
 5 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 94c993303c..78578c42d7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sbuilding\swith\sSQLITE_OMIT_WAL\sdefined.
-D 2017-10-05T20:02:49.526
+C Fix\scompiler\swarnings\sthat\scome\sup\swith\sSQLITE_OMIT_WAL.
+D 2017-10-05T20:57:38.304
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -401,7 +401,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c cc88a7fca7287dfc004921bb5e2764893dfe4f6dd33be3570126b3fc37932600
+F src/btree.c 07ad643c75519359f72622bfb862e53723e0bed52ef7c9979e04a0a531078e34
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
@@ -438,8 +438,8 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
-F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
-F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
+F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
+F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
@@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4258fb578a6e75590c7314fe511ca8dc6659e5f2d3a54d8f60ff705fe51ccc92
-R fba1f72a529b91d37df6bb92e25ac897
-U dan
-Z 4d1d6f0cdfff4b56d945b83bdce453fb
+P 373b0ace480aa303bbf512ea8806a17f6186b16d6316a7b724499bf94b3974d4
+R adacc5e670ad3c34f8576daa184cae71
+U drh
+Z 9327d763f0b9382eb519713049c15b88
diff --git a/manifest.uuid b/manifest.uuid
index d29e532767..9c51f15a9c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-373b0ace480aa303bbf512ea8806a17f6186b16d6316a7b724499bf94b3974d4
\ No newline at end of file
+8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index cadd711e45..1496588f85 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -2922,7 +2922,8 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){
 ** set to the value passed to this function as the second parameter,
 ** set it so.
 */
-#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS
+#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS \
+    && !defined(SQLITE_OMIT_WAL)
 static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){
   sqlite3 *db;
   Db *pDb;
diff --git a/src/os.c b/src/os.c
index 18277df356..26c8065063 100644
--- a/src/os.c
+++ b/src/os.c
@@ -153,6 +153,7 @@ int sqlite3OsSectorSize(sqlite3_file *id){
 int sqlite3OsDeviceCharacteristics(sqlite3_file *id){
   return id->pMethods->xDeviceCharacteristics(id);
 }
+#ifndef SQLITE_OMIT_WAL
 int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){
   return id->pMethods->xShmLock(id, offset, n, flags);
 }
@@ -172,6 +173,7 @@ int sqlite3OsShmMap(
   DO_OS_MALLOC_TEST(id);
   return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp);
 }
+#endif /* SQLITE_OMIT_WAL */
 
 #if SQLITE_MAX_MMAP_SIZE>0
 /* The real implementation of xFetch and xUnfetch */
diff --git a/src/os.h b/src/os.h
index 947f88b36e..f841d6bab1 100644
--- a/src/os.h
+++ b/src/os.h
@@ -174,10 +174,12 @@ void sqlite3OsFileControlHint(sqlite3_file*,int,void*);
 #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0
 int sqlite3OsSectorSize(sqlite3_file *id);
 int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
+#ifndef SQLITE_OMIT_WAL
 int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
 int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
 void sqlite3OsShmBarrier(sqlite3_file *id);
 int sqlite3OsShmUnmap(sqlite3_file *id, int);
+#endif /* SQLITE_OMIT_WAL */
 int sqlite3OsFetch(sqlite3_file *id, i64, int, void **);
 int sqlite3OsUnfetch(sqlite3_file *, i64, void *);
 

From a0ac086f15f7952b5e7865e0c5a9fa15bd208859 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 6 Oct 2017 18:00:36 +0000
Subject: [PATCH 159/270] Add tests for the example fts3 "rank" function that
 appears in the documentation.

FossilOrigin-Name: 702b137aa4f76543647e177beeb1ca2b3cd18c61021c78880e9aa8656f341d65
---
 manifest           |  15 +++---
 manifest.uuid      |   2 +-
 src/test_func.c    | 118 +++++++++++++++++++++++++++++++++++++++++++++
 test/fts3rank.test |  64 ++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 8 deletions(-)
 create mode 100644 test/fts3rank.test

diff --git a/manifest b/manifest
index 78578c42d7..9ed6440a70 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarnings\sthat\scome\sup\swith\sSQLITE_OMIT_WAL.
-D 2017-10-05T20:57:38.304
+C Add\stests\sfor\sthe\sexample\sfts3\s"rank"\sfunction\sthat\sappears\sin\sthe\ndocumentation.
+D 2017-10-06T18:00:36.541
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -489,7 +489,7 @@ F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
 F src/test_fs.c 35a2f7dd8a915900873386331386d9ba1ae1b5026d74fd20c2807bc76221f291
-F src/test_func.c a4fdab3363b436c1b12660e9362ce3f3782b7b5e
+F src/test_func.c 772bb74807d3485ce90c9347f5dfddec4a76ed9edd4cf6c58d2d79b9ce9064d6
 F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d
 F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664
 F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64
@@ -864,6 +864,7 @@ F test/fts3offsets.test b85fd382abdc78ebce721d8117bd552dfb75094c
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
 F test/fts3query.test f33eb71a1fe1084ea585eeb7ee76b390729f5170
+F test/fts3rank.test e4d2e16a28c98cae95001a75e2b4b05b19b051ffd6aaab15491c5e0595127b9b
 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e
 F test/fts3snippet.test 01a4231816e03a0660ae53ba2404fe69012fe0db
@@ -1655,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 373b0ace480aa303bbf512ea8806a17f6186b16d6316a7b724499bf94b3974d4
-R adacc5e670ad3c34f8576daa184cae71
-U drh
-Z 9327d763f0b9382eb519713049c15b88
+P 8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135
+R 413648e1416f08d1828bb75a07c656dd
+U dan
+Z b857c7dffd71ebfaddd17b63d1fd16c3
diff --git a/manifest.uuid b/manifest.uuid
index 9c51f15a9c..b8a9eb4154 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135
\ No newline at end of file
+702b137aa4f76543647e177beeb1ca2b3cd18c61021c78880e9aa8656f341d65
\ No newline at end of file
diff --git a/src/test_func.c b/src/test_func.c
index c7860fe887..59fe677c33 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -791,6 +791,123 @@ abuse_err:
 }
 
 
+/*
+** SQLite user defined function to use with matchinfo() to calculate the
+** relevancy of an FTS match. The value returned is the relevancy score
+** (a real value greater than or equal to zero). A larger value indicates 
+** a more relevant document.
+**
+** The overall relevancy returned is the sum of the relevancies of each 
+** column value in the FTS table. The relevancy of a column value is the
+** sum of the following for each reportable phrase in the FTS query:
+**
+**   (<hit count> / <global hit count>) * <column weight>
+**
+** where <hit count> is the number of instances of the phrase in the
+** column value of the current row and <global hit count> is the number
+** of instances of the phrase in the same column of all rows in the FTS
+** table. The <column weight> is a weighting factor assigned to each
+** column by the caller (see below).
+**
+** The first argument to this function must be the return value of the FTS 
+** matchinfo() function. Following this must be one argument for each column 
+** of the FTS table containing a numeric weight factor for the corresponding 
+** column. Example:
+**
+**     CREATE VIRTUAL TABLE documents USING fts3(title, content)
+**
+** The following query returns the docids of documents that match the full-text
+** query <query> sorted from most to least relevant. When calculating
+** relevance, query term instances in the 'title' column are given twice the
+** weighting of those in the 'content' column.
+**
+**     SELECT docid FROM documents 
+**     WHERE documents MATCH <query> 
+**     ORDER BY rank(matchinfo(documents), 1.0, 0.5) DESC
+*/
+static void rankfunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
+  int *aMatchinfo;                /* Return value of matchinfo() */
+  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
+  int nCol = 0;                   /* Number of columns in the table */
+  int nPhrase = 0;                /* Number of phrases in the query */
+  int iPhrase;                    /* Current phrase */
+  double score = 0.0;             /* Value to return */
+
+  assert( sizeof(int)==4 );
+
+  /* Check that the number of arguments passed to this function is correct.
+  ** If not, jump to wrong_number_args. Set aMatchinfo to point to the array
+  ** of unsigned integer values returned by FTS function matchinfo. Set
+  ** nPhrase to contain the number of reportable phrases in the users full-text
+  ** query, and nCol to the number of columns in the table. Then check that the
+  ** size of the matchinfo blob is as expected. Return an error if it is not.
+  */
+  if( nVal<1 ) goto wrong_number_args;
+  aMatchinfo = (unsigned int *)sqlite3_value_blob(apVal[0]);
+  nMatchinfo = sqlite3_value_bytes(apVal[0]) / sizeof(int);
+  if( nMatchinfo>=2 ){
+    nPhrase = aMatchinfo[0];
+    nCol = aMatchinfo[1];
+  }
+  if( nMatchinfo!=(2+3*nCol*nPhrase) ){
+    sqlite3_result_error(pCtx,
+        "invalid matchinfo blob passed to function rank()", -1);
+    return;
+  }
+  if( nVal!=(1+nCol) ) goto wrong_number_args;
+
+  /* Iterate through each phrase in the users query. */
+  for(iPhrase=0; iPhrase<nPhrase; iPhrase++){
+    int iCol;                     /* Current column */
+
+    /* Now iterate through each column in the users query. For each column,
+    ** increment the relevancy score by:
+    **
+    **   (<hit count> / <global hit count>) * <column weight>
+    **
+    ** aPhraseinfo[] points to the start of the data for phrase iPhrase. So
+    ** the hit count and global hit counts for each column are found in 
+    ** aPhraseinfo[iCol*3] and aPhraseinfo[iCol*3+1], respectively.
+    */
+    int *aPhraseinfo = &aMatchinfo[2 + iPhrase*nCol*3];
+    for(iCol=0; iCol<nCol; iCol++){
+      int nHitCount = aPhraseinfo[3*iCol];
+      int nGlobalHitCount = aPhraseinfo[3*iCol+1];
+      double weight = sqlite3_value_double(apVal[iCol+1]);
+      if( nHitCount>0 ){
+        score += ((double)nHitCount / (double)nGlobalHitCount) * weight;
+      }
+    }
+  }
+
+  sqlite3_result_double(pCtx, score);
+  return;
+
+  /* Jump here if the wrong number of arguments are passed to this function */
+wrong_number_args:
+  sqlite3_result_error(pCtx, "wrong number of arguments to function rank()", -1);
+}
+
+static int SQLITE_TCLAPI install_fts3_rank_function(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**);
+  sqlite3 *db;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  sqlite3_create_function(db, "rank", -1, SQLITE_UTF8, 0, rankfunc, 0, 0);
+  return TCL_OK;
+}
+
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -801,6 +918,7 @@ int Sqlitetest_func_Init(Tcl_Interp *interp){
   } aObjCmd[] = {
      { "autoinstall_test_functions",    autoinstall_test_funcs },
      { "abuse_create_function",         abuse_create_function  },
+     { "install_fts3_rank_function",    install_fts3_rank_function  },
   };
   int i;
   extern int Md5_Register(sqlite3 *, char **, const sqlite3_api_routines *);
diff --git a/test/fts3rank.test b/test/fts3rank.test
new file mode 100644
index 0000000000..7ee3143a76
--- /dev/null
+++ b/test/fts3rank.test
@@ -0,0 +1,64 @@
+# 2017 October 7
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3expr5
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+install_fts3_rank_function db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts3(a, b);
+  INSERT INTO t1 VALUES('one two', 'one');
+  INSERT INTO t1 VALUES('one two', 'three');
+  INSERT INTO t1 VALUES('one two', 'two');
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1 WHERE t1 MATCH 'one' 
+  ORDER BY rank(matchinfo(t1), 1.0, 1.0) DESC, rowid
+} {
+  {one two} one
+  {one two} three
+  {one two} two
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1 WHERE t1 MATCH 'two' 
+  ORDER BY rank(matchinfo(t1), 1.0, 1.0) DESC, rowid
+} {
+  {one two} two
+  {one two} one
+  {one two} three
+}
+
+do_catchsql_test 1.3 {
+  SELECT * FROM t1 ORDER BY rank(matchinfo(t1), 1.0, 1.0) DESC, rowid
+} {1 {invalid matchinfo blob passed to function rank()}}
+
+do_catchsql_test 1.4 {
+  SELECT * FROM t1 ORDER BY rank(x'0000000000000000') DESC, rowid
+} {0 {{one two} one {one two} three {one two} two}}
+
+do_catchsql_test 1.5 {
+  SELECT * FROM t1 ORDER BY rank(x'0100000001000000') DESC, rowid
+} {1 {invalid matchinfo blob passed to function rank()}}
+
+finish_test
+

From b60424e495ab486565fecc11a88863dd5a38c915 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 7 Oct 2017 23:31:33 +0000
Subject: [PATCH 160/270] Fix C99-style variable declaration issue seen with
 older versions of MSVC.

FossilOrigin-Name: 14b181e862bd742555c51b87fab59d6f8f6602fe2c18dde3ef3b020779912419
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  5 ++++-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 9ed6440a70..98e6126f23 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sfor\sthe\sexample\sfts3\s"rank"\sfunction\sthat\sappears\sin\sthe\ndocumentation.
-D 2017-10-06T18:00:36.541
+C Fix\sC99-style\svariable\sdeclaration\sissue\sseen\swith\solder\sversions\sof\sMSVC.
+D 2017-10-07T23:31:33.678
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -530,7 +530,7 @@ F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 831a77aaa7aa43005f1c9bf3e9eb6506f4865e1cf99943ccdcd3be5d2dd8a3c7
+F src/vdbeaux.c 15368cf8463c31ecf49de4ccf82b8d41848715edc02298a8f9f61a54c8352d03
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8ca0fa8dfe6a66aea7fc63f15e6f704cb190aa0760a3fec2db5f6bad3861a135
-R 413648e1416f08d1828bb75a07c656dd
-U dan
-Z b857c7dffd71ebfaddd17b63d1fd16c3
+P 702b137aa4f76543647e177beeb1ca2b3cd18c61021c78880e9aa8656f341d65
+R b0502fb9406ba19f89441a8efa35216f
+U mistachkin
+Z 1e54f1aa1bf639c462d6cc376839202e
diff --git a/manifest.uuid b/manifest.uuid
index b8a9eb4154..499518f380 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-702b137aa4f76543647e177beeb1ca2b3cd18c61021c78880e9aa8656f341d65
\ No newline at end of file
+14b181e862bd742555c51b87fab59d6f8f6602fe2c18dde3ef3b020779912419
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 9046922c39..88b38f77bd 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -2868,6 +2868,10 @@ static void vdbeInvokeSqllog(Vdbe *v){
 ** VDBE_MAGIC_INIT.
 */
 int sqlite3VdbeReset(Vdbe *p){
+#ifdef SQLITE_DEBUG
+  int i;
+#endif
+
   sqlite3 *db;
   db = p->db;
 
@@ -2899,7 +2903,6 @@ int sqlite3VdbeReset(Vdbe *p){
 #ifdef SQLITE_DEBUG
   /* Execute assert() statements to ensure that the Vdbe.apCsr[] and 
   ** Vdbe.aMem[] arrays have already been cleaned up.  */
-  int i;
   if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 );
   if( p->aMem ){
     for(i=0; i<p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined );

From 4537f77a53c2ff70b227389542a41fe49e6c487c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 7 Oct 2017 23:35:40 +0000
Subject: [PATCH 161/270] Revise variable declaration moved in the previous
 check-in so sqlite3VdbeReset() is consistent with sqlite3VdbeRewind().

FossilOrigin-Name: 63d1b425ee6a7d2ae782b738d81d98e39342ae7fe7c4fd217a28a0bdafe4f222
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 98e6126f23..b750b7d766 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sC99-style\svariable\sdeclaration\sissue\sseen\swith\solder\sversions\sof\sMSVC.
-D 2017-10-07T23:31:33.678
+C Revise\svariable\sdeclaration\smoved\sin\sthe\sprevious\scheck-in\sso\ssqlite3VdbeReset()\sis\sconsistent\swith\ssqlite3VdbeRewind().
+D 2017-10-07T23:35:40.530
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -530,7 +530,7 @@ F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 15368cf8463c31ecf49de4ccf82b8d41848715edc02298a8f9f61a54c8352d03
+F src/vdbeaux.c c423065d50cee24bc8cba57764f5e9869a1bb920c50907f5dd363ebd7c5aef82
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 702b137aa4f76543647e177beeb1ca2b3cd18c61021c78880e9aa8656f341d65
-R b0502fb9406ba19f89441a8efa35216f
+P 14b181e862bd742555c51b87fab59d6f8f6602fe2c18dde3ef3b020779912419
+R a3d97a8c85e28378254e6db374dde854
 U mistachkin
-Z 1e54f1aa1bf639c462d6cc376839202e
+Z ee383b61e8263625abf43191677f405e
diff --git a/manifest.uuid b/manifest.uuid
index 499518f380..7d4e9a9ed2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-14b181e862bd742555c51b87fab59d6f8f6602fe2c18dde3ef3b020779912419
\ No newline at end of file
+63d1b425ee6a7d2ae782b738d81d98e39342ae7fe7c4fd217a28a0bdafe4f222
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 88b38f77bd..6ec8b30680 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -2868,7 +2868,7 @@ static void vdbeInvokeSqllog(Vdbe *v){
 ** VDBE_MAGIC_INIT.
 */
 int sqlite3VdbeReset(Vdbe *p){
-#ifdef SQLITE_DEBUG
+#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
   int i;
 #endif
 
@@ -2918,7 +2918,6 @@ int sqlite3VdbeReset(Vdbe *p){
   {
     FILE *out = fopen("vdbe_profile.out", "a");
     if( out ){
-      int i;
       fprintf(out, "---- ");
       for(i=0; i<p->nOp; i++){
         fprintf(out, "%02x", p->aOp[i].opcode);

From cc24f815762dfac9b5a5657027919bd26bf298cd Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 7 Oct 2017 23:58:55 +0000
Subject: [PATCH 162/270] Fix harmless compiler warning seen with MSVC.

FossilOrigin-Name: 39d920d1ef0cce40195b21e148f78f544710348fa180c0e76f743a73e5236d45
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/test_func.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b750b7d766..94f09ae3bf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Revise\svariable\sdeclaration\smoved\sin\sthe\sprevious\scheck-in\sso\ssqlite3VdbeReset()\sis\sconsistent\swith\ssqlite3VdbeRewind().
-D 2017-10-07T23:35:40.530
+C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC.
+D 2017-10-07T23:58:55.057
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -489,7 +489,7 @@ F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
 F src/test_fs.c 35a2f7dd8a915900873386331386d9ba1ae1b5026d74fd20c2807bc76221f291
-F src/test_func.c 772bb74807d3485ce90c9347f5dfddec4a76ed9edd4cf6c58d2d79b9ce9064d6
+F src/test_func.c d12d805953bcb3bb19f71d29cdc93383b7b7a3369504d2b7e398a1bd77376294
 F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d
 F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664
 F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 14b181e862bd742555c51b87fab59d6f8f6602fe2c18dde3ef3b020779912419
-R a3d97a8c85e28378254e6db374dde854
+P 63d1b425ee6a7d2ae782b738d81d98e39342ae7fe7c4fd217a28a0bdafe4f222
+R aa4393f5910ab347fda44397cfe57042
 U mistachkin
-Z ee383b61e8263625abf43191677f405e
+Z c22b12679dc2652828389d7408722be7
diff --git a/manifest.uuid b/manifest.uuid
index 7d4e9a9ed2..6c2e700c6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-63d1b425ee6a7d2ae782b738d81d98e39342ae7fe7c4fd217a28a0bdafe4f222
\ No newline at end of file
+39d920d1ef0cce40195b21e148f78f544710348fa180c0e76f743a73e5236d45
\ No newline at end of file
diff --git a/src/test_func.c b/src/test_func.c
index 59fe677c33..2a7103f88e 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -843,7 +843,7 @@ static void rankfunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
   ** size of the matchinfo blob is as expected. Return an error if it is not.
   */
   if( nVal<1 ) goto wrong_number_args;
-  aMatchinfo = (unsigned int *)sqlite3_value_blob(apVal[0]);
+  aMatchinfo = (int*)sqlite3_value_blob(apVal[0]);
   nMatchinfo = sqlite3_value_bytes(apVal[0]) / sizeof(int);
   if( nMatchinfo>=2 ){
     nPhrase = aMatchinfo[0];

From 94ea27f8086af9ba9d3b529c1445f006f4d4f487 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sun, 8 Oct 2017 02:22:06 +0000
Subject: [PATCH 163/270] Fix harmless compiler warnings in FTS5 seen with
 MSVC.

FossilOrigin-Name: 5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
---
 ext/fts5/fts5_vocab.c |  3 +--
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c
index e48a54c9ab..27f2ff3b2c 100644
--- a/ext/fts5/fts5_vocab.c
+++ b/ext/fts5/fts5_vocab.c
@@ -683,7 +683,6 @@ static int fts5VocabColumnMethod(
       iVal = pCsr->aCnt[0];
     }
   }else{
-    int eDetail = pCsr->pConfig->eDetail;
     assert( eType==FTS5_VOCAB_INSTANCE );
     switch( iCol ){
       case 1:
@@ -694,7 +693,7 @@ static int fts5VocabColumnMethod(
         if( eDetail==FTS5_DETAIL_FULL ){
           ii = FTS5_POS2COLUMN(pCsr->iInstPos);
         }else if( eDetail==FTS5_DETAIL_COLUMNS ){
-          ii = pCsr->iInstPos;
+          ii = (int)pCsr->iInstPos;
         }
         if( ii>=0 && ii<pCsr->pConfig->nCol ){
           const char *z = pCsr->pConfig->azCol[ii];
diff --git a/manifest b/manifest
index 94f09ae3bf..859cd710c0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC.
-D 2017-10-07T23:58:55.057
+C Fix\sharmless\scompiler\swarnings\sin\sFTS5\sseen\swith\sMSVC.
+D 2017-10-08T02:22:06.059
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -114,7 +114,7 @@ F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086aba
 F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
-F ext/fts5/fts5_vocab.c 90783d59cb8ee29ae08ac7f7e1f9c04cc4fb3ffc46d34fedba96d145636dd39d
+F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed0e7a77a6
 F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 63d1b425ee6a7d2ae782b738d81d98e39342ae7fe7c4fd217a28a0bdafe4f222
-R aa4393f5910ab347fda44397cfe57042
+P 39d920d1ef0cce40195b21e148f78f544710348fa180c0e76f743a73e5236d45
+R f768d25ae3366d16350abf238bbc60dc
 U mistachkin
-Z c22b12679dc2652828389d7408722be7
+Z 0e12f4ab1427b4952396ef9a1f9de008
diff --git a/manifest.uuid b/manifest.uuid
index 6c2e700c6b..add2ac521c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-39d920d1ef0cce40195b21e148f78f544710348fa180c0e76f743a73e5236d45
\ No newline at end of file
+5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
\ No newline at end of file

From a2c41c2f2b8cd8f883c3d8d39ec96bdebf5e3bd2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 10 Oct 2017 13:37:22 +0000
Subject: [PATCH 164/270] DROP TABLE now works for WITHOUT ROWID tables and the
 SQLITE_OMIT_AUTOVACUUM compile-time option.  Fix for ticket [e651b1341861d6]

FossilOrigin-Name: b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/build.c   |  9 ---------
 3 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 859cd710c0..a7b54adaba 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\sFTS5\sseen\swith\sMSVC.
-D 2017-10-08T02:22:06.059
+C DROP\sTABLE\snow\sworks\sfor\sWITHOUT\sROWID\stables\sand\sthe\sSQLITE_OMIT_AUTOVACUUM\ncompile-time\soption.\s\sFix\sfor\sticket\s[e651b1341861d6]
+D 2017-10-10T13:37:22.404
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -404,7 +404,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 07ad643c75519359f72622bfb862e53723e0bed52ef7c9979e04a0a531078e34
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c e71e96a67daf3d1dd23188423e66cd6af38017e2ec73fead5d2b57da2d3c7e16
+F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39d920d1ef0cce40195b21e148f78f544710348fa180c0e76f743a73e5236d45
-R f768d25ae3366d16350abf238bbc60dc
-U mistachkin
-Z 0e12f4ab1427b4952396ef9a1f9de008
+P 5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
+R f3d3a32072dba092302fd2943ac3dda5
+U drh
+Z c352a62dec18b44931aef87cbd4d711d
diff --git a/manifest.uuid b/manifest.uuid
index add2ac521c..a83c526cd7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
\ No newline at end of file
+b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 6cd23c2abf..cb8c374d1a 100644
--- a/src/build.c
+++ b/src/build.c
@@ -2366,14 +2366,6 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){
 ** is also added (this can happen with an auto-vacuum database).
 */
 static void destroyTable(Parse *pParse, Table *pTab){
-#ifdef SQLITE_OMIT_AUTOVACUUM
-  Index *pIdx;
-  int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
-  destroyRootPage(pParse, pTab->tnum, iDb);
-  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-    destroyRootPage(pParse, pIdx->tnum, iDb);
-  }
-#else
   /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM
   ** is not defined), then it is important to call OP_Destroy on the
   ** table and index root-pages in order, starting with the numerically 
@@ -2416,7 +2408,6 @@ static void destroyTable(Parse *pParse, Table *pTab){
       iDestroyed = iLargest;
     }
   }
-#endif
 }
 
 /*

From 679afb492ec617d04ee191d5fdccd8e66efb4b17 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 11:12:20 +0000
Subject: [PATCH 165/270] Create a branch for the 3.21.0 release.  Development
 continues on trunk.

FossilOrigin-Name: 0f160a8abb0409b6e687ebc13b732559b3d65428f72430df142fd925eb39e9a2
---
 manifest      | 11 +++++++----
 manifest.uuid |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/manifest b/manifest
index a7b54adaba..fb182d0115 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C DROP\sTABLE\snow\sworks\sfor\sWITHOUT\sROWID\stables\sand\sthe\sSQLITE_OMIT_AUTOVACUUM\ncompile-time\soption.\s\sFix\sfor\sticket\s[e651b1341861d6]
-D 2017-10-10T13:37:22.404
+C Create\sa\sbranch\sfor\sthe\s3.21.0\srelease.\s\sDevelopment\scontinues\son\strunk.
+D 2017-10-11T11:12:20.240
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1656,7 +1656,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
+P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
 R f3d3a32072dba092302fd2943ac3dda5
+T *branch * branch-3.21
+T *sym-branch-3.21 *
+T -sym-trunk *
 U drh
-Z c352a62dec18b44931aef87cbd4d711d
+Z 2b6b9e30a8a2f29d1c1889fa41916c23
diff --git a/manifest.uuid b/manifest.uuid
index a83c526cd7..dc796a5a81 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
\ No newline at end of file
+0f160a8abb0409b6e687ebc13b732559b3d65428f72430df142fd925eb39e9a2
\ No newline at end of file

From ff0a67a1a7d05c16d9411055f6a834c9da3e14a5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 12:20:36 +0000
Subject: [PATCH 166/270] In the speed-check.sh test script, allow an
 additional test-name argument which becomes the comparison baseline, in place
 of "trunk".

FossilOrigin-Name: 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/speed-check.sh | 12 ++++++++----
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index a7b54adaba..b8e6ca811e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C DROP\sTABLE\snow\sworks\sfor\sWITHOUT\sROWID\stables\sand\sthe\sSQLITE_OMIT_AUTOVACUUM\ncompile-time\soption.\s\sFix\sfor\sticket\s[e651b1341861d6]
-D 2017-10-10T13:37:22.404
+C In\sthe\sspeed-check.sh\stest\sscript,\sallow\san\sadditional\stest-name\sargument\nwhich\sbecomes\sthe\scomparison\sbaseline,\sin\splace\sof\s"trunk".
+D 2017-10-11T12:20:36.082
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1616,7 +1616,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl f40dc82b4d5e39d040a02a3ec38268e324068815e4292a15ffa30ee93208bbfd
-F tool/speed-check.sh 9eccb9ade8806238a4e9d6cb6511e7be2f64aff6873c41ea70d322219ea28adf
+F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5594a121bf132a98d0ecb4cf86d9f2681925c9416206096bf11c9370a5dae22f
-R f3d3a32072dba092302fd2943ac3dda5
+P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
+R 9a03aa36b167c86705816bfaa31cdc19
 U drh
-Z c352a62dec18b44931aef87cbd4d711d
+Z 8f5e2f1e221681e7d4bf20eb32b86754
diff --git a/manifest.uuid b/manifest.uuid
index a83c526cd7..7305d2515a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
\ No newline at end of file
+0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
\ No newline at end of file
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
index f9efb7c13d..2cda5c8078 100644
--- a/tool/speed-check.sh
+++ b/tool/speed-check.sh
@@ -29,13 +29,14 @@ SIZE=5
 LEAN_OPTS="-DSQLITE_THREADSAFE=0"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOB"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
+BASELINE="trunk"
 doExplain=0
 doCachegrind=1
 while test "$1" != ""; do
@@ -116,9 +117,12 @@ while test "$1" != ""; do
     --orm)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm"
         ;;
-    *)
+    -*)
         CC_OPTS="$CC_OPTS $1"
         ;;
+    *)
+	BASELINE=$1
+        ;;
   esac
   shift
 done
@@ -153,6 +157,6 @@ fi
 if test $doExplain -eq 1; then
   ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
 fi
-if test "$NAME" != "trunk"; then
-  fossil test-diff --tk -c 20 cout-trunk.txt cout-$NAME.txt
+if test "$NAME" != "$BASELINE"; then
+  fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt
 fi

From a43c8c8a6065b3eed4b1a7ec03a7535f3b3b537f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 13:48:11 +0000
Subject: [PATCH 167/270] Initial implementation of the "sqlite_dbpage" virtual
 table.  Currently it is read-only and has a place-holder xBestIndex.

FossilOrigin-Name: c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49
---
 Makefile.in         |   8 +-
 Makefile.msc        |   9 +-
 main.mk             |   5 +-
 manifest            |  26 +++--
 manifest.uuid       |   2 +-
 src/dbpage.c        | 241 ++++++++++++++++++++++++++++++++++++++++++++
 src/main.c          |   6 ++
 src/sqliteInt.h     |   3 +
 tool/mksqlite3c.tcl |   1 +
 9 files changed, 285 insertions(+), 16 deletions(-)
 create mode 100644 src/dbpage.c

diff --git a/Makefile.in b/Makefile.in
index 58c2c2919b..7ea97e6195 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -166,7 +166,8 @@ USE_AMALGAMATION = @USE_AMALGAMATION@
 #
 LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -215,6 +216,7 @@ SRC = \
   $(TOP)/src/complete.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/delete.c \
   $(TOP)/src/expr.c \
@@ -449,6 +451,7 @@ TESTSRC2 = \
   $(TOP)/src/build.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/expr.c \
   $(TOP)/src/func.c \
@@ -752,6 +755,9 @@ ctime.lo:	$(TOP)/src/ctime.c $(HDR)
 date.lo:	$(TOP)/src/date.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c
 
+dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
+
 dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c
 
diff --git a/Makefile.msc b/Makefile.msc
index 10585e4a58..d59997388d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1091,7 +1091,8 @@ LTLIBS = $(LTLIBS) $(LIBICU)
 #
 LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
-         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
+         callback.lo complete.lo ctime.lo \
+         date.lo dbpage.lo dbstat.lo delete.lo \
          expr.lo fault.lo fkey.lo \
          fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
          fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
@@ -1154,6 +1155,7 @@ SRC00 = \
   $(TOP)\src\complete.c \
   $(TOP)\src\ctime.c \
   $(TOP)\src\date.c \
+  $(TOP)\src\dbpage.c \
   $(TOP)\src\dbstat.c \
   $(TOP)\src\delete.c \
   $(TOP)\src\expr.c \
@@ -1747,7 +1749,10 @@ ctime.lo:	$(TOP)\src\ctime.c $(HDR)
 date.lo:	$(TOP)\src\date.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c
 
-dbstat.lo:	$(TOP)\src\date.c $(HDR)
+dbpage.lo:	$(TOP)\src\dbpage.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c
+
+dbstat.lo:	$(TOP)\src\dbstat.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c
 
 delete.lo:	$(TOP)\src\delete.c $(HDR)
diff --git a/main.mk b/main.mk
index 7da6db15f1..3f83197d4d 100644
--- a/main.mk
+++ b/main.mk
@@ -55,7 +55,8 @@ THREADLIB += $(LIBS)
 LIBOBJ+= vdbe.o parse.o \
          alter.o analyze.o attach.o auth.o \
          backup.o bitvec.o btmutex.o btree.o build.o \
-         callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o \
+         callback.o complete.o ctime.o \
+         date.o dbpage.o dbstat.o delete.o expr.o \
 	 fault.o fkey.o \
          fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
          fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \
@@ -96,6 +97,7 @@ SRC = \
   $(TOP)/src/complete.c \
   $(TOP)/src/ctime.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/delete.c \
   $(TOP)/src/expr.c \
@@ -359,6 +361,7 @@ TESTSRC2 = \
   $(TOP)/src/btree.c \
   $(TOP)/src/build.c \
   $(TOP)/src/date.c \
+  $(TOP)/src/dbpage.c \
   $(TOP)/src/dbstat.c \
   $(TOP)/src/expr.c \
   $(TOP)/src/func.c \
diff --git a/manifest b/manifest
index b8e6ca811e..dbf72466e4 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C In\sthe\sspeed-check.sh\stest\sscript,\sallow\san\sadditional\stest-name\sargument\nwhich\sbecomes\sthe\scomparison\sbaseline,\sin\splace\sof\s"trunk".
-D 2017-10-11T12:20:36.082
-F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
+C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex.
+D 2017-10-11T13:48:11.877
+F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
+F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222
+F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -409,6 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
+F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a
@@ -423,7 +424,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c a4bdadaaa827e7380cba4de878ed7947dab5aeb84f617118ba6a0422cd745b4b
+F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -464,7 +465,7 @@ F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
-F src/sqliteInt.h c07bc88eca1f59ce73e1f486187d0df4effe67c4579e112dfdd91c159e5c0569
+F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1597,7 +1598,7 @@ F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
-F tool/mksqlite3c.tcl b258d679829a9305f5cf107b7d97b9bf23adb3773df42947fed5ef7b180dfbd9
+F tool/mksqlite3c.tcl 1fb69d39166f52d802a70ec37d99bca51d011c8ab30be27bc495be493196ae41
 F tool/mksqlite3h.tcl f92f994d9709aeb9e2b6e6f9fc8b069d2f55202c8e23f453edc44390a25982dc
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
@@ -1656,7 +1657,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
-R 9a03aa36b167c86705816bfaa31cdc19
+P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
+R 9d91f38d6de2e0aabd371a2c4dbcd832
+T *branch * dbpage
+T *sym-dbpage *
+T -sym-trunk *
 U drh
-Z 8f5e2f1e221681e7d4bf20eb32b86754
+Z 95e75b385006fb35e51e104eb16c88ba
diff --git a/manifest.uuid b/manifest.uuid
index 7305d2515a..b0152fffa9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
\ No newline at end of file
+c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
new file mode 100644
index 0000000000..89dcf04227
--- /dev/null
+++ b/src/dbpage.c
@@ -0,0 +1,241 @@
+/*
+** 2017-10-11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of the "sqlite_dbpage" virtual table.
+**
+** The sqlite_dbpage virtual table is used to read or write whole raw
+** pages of the database file.  The pager interface is used so that 
+** uncommitted changes and changes recorded in the WAL file are correctly
+** retrieved.
+*/
+
+#include "sqliteInt.h"   /* Requires access to internal data structures */
+#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \
+    && !defined(SQLITE_OMIT_VIRTUALTABLE)
+
+typedef struct DbpageTable DbpageTable;
+typedef struct DbpageCursor DbpageCursor;
+
+struct DbpageCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  int pgno;                       /* Current page number */
+};
+
+struct DbpageTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;                    /* The database */
+  Pager *pPager;                  /* Pager being read/written */
+  int iDb;                        /* Index of database to analyze */
+  int szPage;                     /* Size of each page in bytes */
+  int nPage;                      /* Number of pages in the file */
+};
+
+/*
+** Connect to or create a dbpagevfs virtual table.
+*/
+static int dbpageConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  DbpageTable *pTab = 0;
+  int rc = SQLITE_OK;
+  int iDb;
+
+  if( argc>=4 ){
+    Token nm;
+    sqlite3TokenInit(&nm, (char*)argv[3]);
+    iDb = sqlite3FindDb(db, &nm);
+    if( iDb<0 ){
+      *pzErr = sqlite3_mprintf("no such schema: %s", argv[3]);
+      return SQLITE_ERROR;
+    }
+  }else{
+    iDb = 0;
+  }
+  rc = sqlite3_declare_vtab(db, 
+          "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)");
+  if( rc==SQLITE_OK ){
+    pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM_BKPT;
+  }
+
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( rc==SQLITE_OK ){
+    Btree *pBt = db->aDb[iDb].pBt;
+    memset(pTab, 0, sizeof(DbpageTable));
+    pTab->db = db;
+    pTab->iDb = iDb;
+    pTab->pPager = pBt ? sqlite3BtreePager(pBt) : 0;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a dbpagevfs virtual table.
+*/
+static int dbpageDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** idxNum:
+**
+**     0     full table scan
+**     1     pgno=?1
+*/
+static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
+  return SQLITE_OK;
+}
+
+/*
+** Open a new dbpagevfs cursor.
+*/
+static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  DbpageCursor *pCsr;
+
+  pCsr = (DbpageCursor *)sqlite3_malloc64(sizeof(DbpageCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM_BKPT;
+  }else{
+    memset(pCsr, 0, sizeof(DbpageCursor));
+    pCsr->base.pVtab = pVTab;
+    pCsr->pgno = -1;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a dbpagevfs cursor.
+*/
+static int dbpageClose(sqlite3_vtab_cursor *pCursor){
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Move a dbpagevfs cursor to the next entry in the file.
+*/
+static int dbpageNext(sqlite3_vtab_cursor *pCursor){
+  int rc = SQLITE_OK;
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  pCsr->pgno++;
+  return rc;
+}
+
+static int dbpageEof(sqlite3_vtab_cursor *pCursor){
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
+  return pCsr->pgno >= pTab->nPage;
+}
+
+static int dbpageFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
+  int rc = SQLITE_OK;
+  Btree *pBt = pTab->db->aDb[pTab->iDb].pBt;
+
+  if( idxNum==1 ){
+    pCsr->pgno = sqlite3_value_int(argv[0]);
+  }else{
+    pCsr->pgno = 0;
+  }
+  pTab->szPage = sqlite3BtreeGetPageSize(pBt);
+  pTab->nPage = sqlite3BtreeLastPage(pBt);
+  return rc;
+}
+
+static int dbpageColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
+  int rc = SQLITE_OK;
+  switch( i ){
+    case 0: {           /* pgno */
+      sqlite3_result_int(ctx, pCsr->pgno);
+      break;
+    }
+    case 1: {           /* data */
+      DbPage *pDbPage = 0;
+      rc = sqlite3PagerGet(pTab->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0);
+      if( rc==SQLITE_OK ){
+        sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pTab->szPage,
+                            SQLITE_TRANSIENT);
+      }
+      sqlite3PagerUnref(pDbPage);
+      break;
+    }
+    default: {          /* schema */
+      sqlite3 *db = sqlite3_context_db_handle(ctx);
+      sqlite3_result_text(ctx, db->aDb[pTab->iDb].zDbSName, -1, SQLITE_STATIC);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  *pRowid = pCsr->pgno;
+  return SQLITE_OK;
+}
+
+/*
+** Invoke this routine to register the "dbpage" virtual table module
+*/
+int sqlite3DbpageRegister(sqlite3 *db){
+  static sqlite3_module dbpage_module = {
+    0,                            /* iVersion */
+    dbpageConnect,                /* xCreate */
+    dbpageConnect,                /* xConnect */
+    dbpageBestIndex,              /* xBestIndex */
+    dbpageDisconnect,             /* xDisconnect */
+    dbpageDisconnect,             /* xDestroy */
+    dbpageOpen,                   /* xOpen - open a cursor */
+    dbpageClose,                  /* xClose - close a cursor */
+    dbpageFilter,                 /* xFilter - configure scan constraints */
+    dbpageNext,                   /* xNext - advance a cursor */
+    dbpageEof,                    /* xEof - check for end of scan */
+    dbpageColumn,                 /* xColumn - read data */
+    dbpageRowid,                  /* xRowid - read data */
+    0,                            /* xUpdate */
+    0,                            /* xBegin */
+    0,                            /* xSync */
+    0,                            /* xCommit */
+    0,                            /* xRollback */
+    0,                            /* xFindMethod */
+    0,                            /* xRename */
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+  };
+  return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0);
+}
+#elif defined(SQLITE_ENABLE_DBPAGE_VTAB)
+int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; }
+#endif /* SQLITE_ENABLE_DBSTAT_VTAB */
diff --git a/src/main.c b/src/main.c
index 3d7609ce5b..49613f6c74 100644
--- a/src/main.c
+++ b/src/main.c
@@ -3054,6 +3054,12 @@ static int openDatabase(
   }
 #endif
 
+#ifdef SQLITE_ENABLE_DBPAGE_VTAB
+  if( !db->mallocFailed && rc==SQLITE_OK){
+    rc = sqlite3DbpageRegister(db);
+  }
+#endif
+
 #ifdef SQLITE_ENABLE_DBSTAT_VTAB
   if( !db->mallocFailed && rc==SQLITE_OK){
     rc = sqlite3DbstatRegister(db);
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index a8f1bed512..0cc435d7b7 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4400,6 +4400,9 @@ int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*);
 int sqlite3ThreadJoin(SQLiteThread*, void**);
 #endif
 
+#if defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)
+int sqlite3DbpageRegister(sqlite3*);
+#endif
 #if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST)
 int sqlite3DbstatRegister(sqlite3*);
 #endif
diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl
index 933819d12b..8ea3e81c91 100644
--- a/tool/mksqlite3c.tcl
+++ b/tool/mksqlite3c.tcl
@@ -394,6 +394,7 @@ foreach file {
    fts3_icu.c
    sqlite3rbu.c
    dbstat.c
+   dbpage.c
    sqlite3session.c
    json1.c
    fts5.c

From 34d0b1ac56e7cdf587dfbb06a136d560032deedf Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 15:02:53 +0000
Subject: [PATCH 168/270] Get writes working on the sqlite_dbpage virtual
 table.  Add a few test cases.

FossilOrigin-Name: a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f
---
 manifest         |  16 ++++---
 manifest.uuid    |   2 +-
 src/dbpage.c     | 106 +++++++++++++++++++++++++++++++++++++++++++----
 test/dbpage.test |  69 ++++++++++++++++++++++++++++++
 4 files changed, 174 insertions(+), 19 deletions(-)
 create mode 100644 test/dbpage.test

diff --git a/manifest b/manifest
index dbf72466e4..3e606d8851 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Initial\simplementation\sof\sthe\s"sqlite_dbpage"\svirtual\stable.\s\sCurrently\nit\sis\sread-only\sand\shas\sa\splace-holder\sxBestIndex.
-D 2017-10-11T13:48:11.877
+C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases.
+D 2017-10-11T15:02:53.047
 F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d
@@ -409,7 +409,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
-F src/dbpage.c f274a9b7bb680cc2952ee78883d67e133be0ef6065317813586a5f723af35ad5
+F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a
@@ -709,6 +709,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b
 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
+F test/dbpage.test 9cf4dc92a4de67c81e5c32b24e3fbb8c4757e4b642694a219b3090a4f9277a4d
 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
@@ -1657,10 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
-R 9d91f38d6de2e0aabd371a2c4dbcd832
-T *branch * dbpage
-T *sym-dbpage *
-T -sym-trunk *
+P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49
+R 92490bca397034b7f3fba4e951f0781d
 U drh
-Z 95e75b385006fb35e51e104eb16c88ba
+Z ddf133cf759c24442aeeb91134927fef
diff --git a/manifest.uuid b/manifest.uuid
index b0152fffa9..7a388fce6d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49
\ No newline at end of file
+a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
index 89dcf04227..d21c5b6df1 100644
--- a/src/dbpage.c
+++ b/src/dbpage.c
@@ -16,6 +16,19 @@
 ** pages of the database file.  The pager interface is used so that 
 ** uncommitted changes and changes recorded in the WAL file are correctly
 ** retrieved.
+**
+** Usage example:
+**
+**    SELECT data FROM sqlite_dbpage('aux1') WHERE pgno=123;
+**
+** This is an eponymous virtual table so it does not need to be created before
+** use.  The optional argument to the sqlite_dbpage() table name is the
+** schema for the database file that is to be read.  The default schema is
+** "main".
+**
+** The data field of sqlite_dbpage table can be updated.  The new
+** value must be a BLOB which is the correct page size, otherwise the
+** update fails.  Rows may not be deleted or inserted.
 */
 
 #include "sqliteInt.h"   /* Requires access to internal data structures */
@@ -28,6 +41,7 @@ typedef struct DbpageCursor DbpageCursor;
 struct DbpageCursor {
   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
   int pgno;                       /* Current page number */
+  int mxPgno;                     /* Last page to visit on this scan */
 };
 
 struct DbpageTable {
@@ -65,7 +79,7 @@ static int dbpageConnect(
     iDb = 0;
   }
   rc = sqlite3_declare_vtab(db, 
-          "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB)");
+          "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)");
   if( rc==SQLITE_OK ){
     pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable));
     if( pTab==0 ) rc = SQLITE_NOMEM_BKPT;
@@ -99,7 +113,26 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){
 **     1     pgno=?1
 */
 static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
   pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      pIdxInfo->estimatedRows = 1;
+      pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE;
+      pIdxInfo->estimatedCost = 1.0;
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      break;
+    }
+  }
+  if( pIdxInfo->nOrderBy>=1
+   && pIdxInfo->aOrderBy[0].iColumn<=0
+   && pIdxInfo->aOrderBy[0].desc==0
+  ){
+    pIdxInfo->orderByConsumed = 1;
+  }
   return SQLITE_OK;
 }
 
@@ -143,8 +176,7 @@ static int dbpageNext(sqlite3_vtab_cursor *pCursor){
 
 static int dbpageEof(sqlite3_vtab_cursor *pCursor){
   DbpageCursor *pCsr = (DbpageCursor *)pCursor;
-  DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
-  return pCsr->pgno >= pTab->nPage;
+  return pCsr->pgno > pCsr->mxPgno;
 }
 
 static int dbpageFilter(
@@ -157,13 +189,20 @@ static int dbpageFilter(
   int rc = SQLITE_OK;
   Btree *pBt = pTab->db->aDb[pTab->iDb].pBt;
 
-  if( idxNum==1 ){
-    pCsr->pgno = sqlite3_value_int(argv[0]);
-  }else{
-    pCsr->pgno = 0;
-  }
   pTab->szPage = sqlite3BtreeGetPageSize(pBt);
   pTab->nPage = sqlite3BtreeLastPage(pBt);
+  if( idxNum==1 ){
+    pCsr->pgno = sqlite3_value_int(argv[0]);
+    if( pCsr->pgno<1 || pCsr->pgno>pTab->nPage ){
+      pCsr->pgno = 1;
+      pCsr->mxPgno = 0;
+    }else{
+      pCsr->mxPgno = pCsr->pgno;
+    }
+  }else{
+    pCsr->pgno = 1;
+    pCsr->mxPgno = pTab->nPage;
+  }
   return rc;
 }
 
@@ -205,6 +244,55 @@ static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
   return SQLITE_OK;
 }
 
+static int dbpageUpdate(
+  sqlite3_vtab *pVtab,
+  int argc,
+  sqlite3_value **argv,
+  sqlite_int64 *pRowid
+){
+  DbpageTable *pTab = (DbpageTable *)pVtab;
+  int pgno;
+  DbPage *pDbPage = 0;
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+
+  if( argc==1 ){
+    zErr = "cannot delete";
+    goto update_fail;
+  }
+  pgno = sqlite3_value_int(argv[0]);
+  if( pgno<1 || pgno>pTab->nPage ){
+    zErr = "bad page number";
+    goto update_fail;
+  }
+  if( sqlite3_value_int(argv[1])!=pgno ){
+    zErr = "cannot insert";
+    goto update_fail;
+  }
+  if( sqlite3_value_type(argv[3])!=SQLITE_BLOB 
+   || sqlite3_value_bytes(argv[3])!=pTab->szPage 
+  ){
+    zErr = "bad page value";
+    goto update_fail;
+  }
+  rc = sqlite3PagerGet(pTab->pPager, pgno, (DbPage**)&pDbPage, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3PagerWrite(pDbPage);
+    if( rc==SQLITE_OK ){
+      memcpy(sqlite3PagerGetData(pDbPage),
+             sqlite3_value_blob(argv[3]),
+             pTab->szPage);
+    }
+  }
+  sqlite3PagerUnref(pDbPage);
+  return rc;
+
+update_fail:
+  sqlite3_free(pVtab->zErrMsg);
+  pVtab->zErrMsg = sqlite3_mprintf("%s", zErr);
+  return SQLITE_ERROR;
+}
+
 /*
 ** Invoke this routine to register the "dbpage" virtual table module
 */
@@ -223,7 +311,7 @@ int sqlite3DbpageRegister(sqlite3 *db){
     dbpageEof,                    /* xEof - check for end of scan */
     dbpageColumn,                 /* xColumn - read data */
     dbpageRowid,                  /* xRowid - read data */
-    0,                            /* xUpdate */
+    dbpageUpdate,                 /* xUpdate */
     0,                            /* xBegin */
     0,                            /* xSync */
     0,                            /* xCommit */
diff --git a/test/dbpage.test b/test/dbpage.test
new file mode 100644
index 0000000000..e29d4b33a9
--- /dev/null
+++ b/test/dbpage.test
@@ -0,0 +1,69 @@
+# 2017-10-11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the sqlite_dbpage virtual table.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix dbpage
+
+ifcapable !vtab||!compound {
+  finish_test
+  return
+}
+
+do_execsql_test 100 {
+  PRAGMA page_size=4096;
+  PRAGMA journal_mode=WAL;
+  CREATE TABLE t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c;
+  PRAGMA integrity_check;
+} {wal ok}
+do_execsql_test 110 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno;
+} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0D00000016'}
+do_execsql_test 120 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=2;
+} {2 X'0500000001'}
+do_execsql_test 130 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=4;
+} {4 X'0D00000016'}
+do_execsql_test 140 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=5;
+} {}
+do_execsql_test 150 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=0;
+} {}
+
+do_execsql_test 200 {
+  CREATE TEMP TABLE saved_content(x);
+  INSERT INTO saved_content(x) SELECT data FROM sqlite_dbpage WHERE pgno=4;
+  UPDATE sqlite_dbpage SET data=zeroblob(4096) WHERE pgno=4;
+} {}
+do_catchsql_test 210 {
+  PRAGMA integrity_check;
+} {1 {database disk image is malformed}}
+do_execsql_test 220 {
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno;
+} {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0000000000'}
+do_execsql_test 230 {
+  UPDATE sqlite_dbpage SET data=(SELECT x FROM saved_content) WHERE pgno=4;
+} {}
+do_catchsql_test 230 {
+  PRAGMA integrity_check;
+} {0 ok}
+
+
+
+
+finish_test

From ca5cf1229be7c3e60c6a2daf2c01feba6d9aeb2e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 17:13:29 +0000
Subject: [PATCH 169/270] Enable sqlite_dbpage and dbstat virtual tables in the
 shell, by default.

FossilOrigin-Name: 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db
---
 Makefile.in   |  2 ++
 Makefile.msc  |  3 ++-
 main.mk       |  2 ++
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 5 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 7ea97e6195..5e3f14279e 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -572,6 +572,8 @@ SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/Makefile.msc b/Makefile.msc
index d59997388d..febf496ee3 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1510,7 +1510,8 @@ FUZZDATA = \
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 !ENDIF
 
 # <<mark>>
diff --git a/main.mk b/main.mk
index 3f83197d4d..26a0bd586d 100644
--- a/main.mk
+++ b/main.mk
@@ -483,6 +483,8 @@ SHELL_OPT += -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/manifest b/manifest
index 3e606d8851..e250415fef 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Get\swrites\sworking\son\sthe\ssqlite_dbpage\svirtual\stable.\s\sAdd\sa\sfew\stest\scases.
-D 2017-10-11T15:02:53.047
-F Makefile.in f7cba589198b8663d8a43f25ad001cf44fdac4fcd6216325f05775924a7af2f9
+C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault.
+D 2017-10-11T17:13:29.048
+F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 1224c8773fc7f89c57e83c7b452291d239aa2cff4af50a204c84129e295cc37d
+F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 656d9f31fdc01290094460f00255a45fdf4bedd08839467857919fea628582cf
+F main.mk dc4d9dfe050e6d65671cb940974bd6b76451287db334c47332552ede28325714
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c2c1d656e3f52465192c2a697a976cd1837ccc4e10708a2377cff8bf6eaa7d49
-R 92490bca397034b7f3fba4e951f0781d
+P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f
+R 26ea83977ebe8e13c118f8a4befc3e5d
 U drh
-Z ddf133cf759c24442aeeb91134927fef
+Z d68d7c86064c38dfe4b5e452d55c3294
diff --git a/manifest.uuid b/manifest.uuid
index 7a388fce6d..4b74c0454f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f
\ No newline at end of file
+01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db
\ No newline at end of file

From 9b40ab45b234fdc7a28fc22b565d63fce4982311 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 17:25:08 +0000
Subject: [PATCH 170/270] Enable JSON by default in the Windows command-line
 shell sqlite3.exe.

FossilOrigin-Name: 79c96121baf065fd78ba4f1fe82ce725b6372df20e9323d88022b7c243701372
---
 Makefile.msc  |  2 +-
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 10585e4a58..de4f8b196f 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1508,7 +1508,7 @@ FUZZDATA = \
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 !ENDIF
 
 # <<mark>>
diff --git a/manifest b/manifest
index fb182d0115..6e831045b9 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Create\sa\sbranch\sfor\sthe\s3.21.0\srelease.\s\sDevelopment\scontinues\son\strunk.
-D 2017-10-11T11:12:20.240
+C Enable\sJSON\sby\sdefault\sin\sthe\sWindows\scommand-line\sshell\ssqlite3.exe.
+D 2017-10-11T17:25:08.609
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
+F Makefile.msc aade743c45bf10cc6d0150d97842e310ac3d91d4f3640a286139d4e72451b96f
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1656,10 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
-R f3d3a32072dba092302fd2943ac3dda5
-T *branch * branch-3.21
-T *sym-branch-3.21 *
-T -sym-trunk *
+P 0f160a8abb0409b6e687ebc13b732559b3d65428f72430df142fd925eb39e9a2
+R 7fea6f857c00d4f29f020f0f9a952e04
 U drh
-Z 2b6b9e30a8a2f29d1c1889fa41916c23
+Z b088c8a18ee6c3891400703c6e00fc6b
diff --git a/manifest.uuid b/manifest.uuid
index dc796a5a81..ce3f7ad848 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0f160a8abb0409b6e687ebc13b732559b3d65428f72430df142fd925eb39e9a2
\ No newline at end of file
+79c96121baf065fd78ba4f1fe82ce725b6372df20e9323d88022b7c243701372
\ No newline at end of file

From 690e8858fe65b7ed007a95f2a55512a5ad17285e Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 11 Oct 2017 17:50:22 +0000
Subject: [PATCH 171/270] Make JSON1 defines consistent in the makefiles for
 MSVC.

FossilOrigin-Name: 31eee0e4bffc5419e076a589049bfea7327dfc666fcc2767c0b700f506055c4e
---
 Makefile.msc          |  2 +-
 autoconf/Makefile.msc |  2 +-
 manifest              | 16 ++++++++--------
 manifest.uuid         |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index de4f8b196f..0fed23dce6 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1514,7 +1514,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE
 # <<mark>>
 # Extra compiler options for various test tools.
 #
-MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
+MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index f0f9a01ee0..f535d1450a 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -927,7 +927,7 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 !ENDIF
 
 
diff --git a/manifest b/manifest
index 6e831045b9..804cc5e872 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Enable\sJSON\sby\sdefault\sin\sthe\sWindows\scommand-line\sshell\ssqlite3.exe.
-D 2017-10-11T17:25:08.609
+C Make\sJSON1\sdefines\sconsistent\sin\sthe\smakefiles\sfor\sMSVC.
+D 2017-10-11T17:50:22.538
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc aade743c45bf10cc6d0150d97842e310ac3d91d4f3640a286139d4e72451b96f
+F Makefile.msc 307701b46e4ac0e2aaa7776ea5936fff21636e991c9d5988584f37d65be9c13e
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
-F autoconf/Makefile.msc b77aec100e4fb4739748a2461b5aa82c179fcde35bc0e08ce52ae7322d218701
+F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0f160a8abb0409b6e687ebc13b732559b3d65428f72430df142fd925eb39e9a2
-R 7fea6f857c00d4f29f020f0f9a952e04
-U drh
-Z b088c8a18ee6c3891400703c6e00fc6b
+P 79c96121baf065fd78ba4f1fe82ce725b6372df20e9323d88022b7c243701372
+R 2d35bd3f9bb9d088160d8caa2fcda76b
+U mistachkin
+Z ae3424fdef9b7840b735184e2f02e403
diff --git a/manifest.uuid b/manifest.uuid
index ce3f7ad848..f9b265f962 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-79c96121baf065fd78ba4f1fe82ce725b6372df20e9323d88022b7c243701372
\ No newline at end of file
+31eee0e4bffc5419e076a589049bfea7327dfc666fcc2767c0b700f506055c4e
\ No newline at end of file

From 512e6c3c5118cba59ed3fa66a586e87100b91299 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 17:51:08 +0000
Subject: [PATCH 172/270] Convert the implementation of the ".dbstat"
 dot-command of the command-line shell to use the sqlite_dbpage table.

FossilOrigin-Name: 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c    | 18 +++++++++++-------
 src/shell.c.in | 18 +++++++++++-------
 4 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index e250415fef..92b51678a2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\ssqlite_dbpage\sand\sdbstat\svirtual\stables\sin\sthe\sshell,\sby\sdefault.
-D 2017-10-11T17:13:29.048
+C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable.
+D 2017-10-11T17:51:08.392
 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589
@@ -460,8 +460,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe
-F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0
+F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e
+F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1658,7 +1658,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8b264d811e5bcb7e3ae8a12bf5b6830a9d1adff1f59436dda9e886f97da242f
-R 26ea83977ebe8e13c118f8a4befc3e5d
+P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db
+R 32cd47bb05391f34b12cc1216f3e5e53
 U drh
-Z d68d7c86064c38dfe4b5e452d55c3294
+Z ef5ad8ed0ad3bdbe43b1de17eb74fe58
diff --git a/manifest.uuid b/manifest.uuid
index 4b74c0454f..12bf79e9d8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db
\ No newline at end of file
+497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index f7be941570..2b77d482e6 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -4970,20 +4970,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
      { "schema size:",
        "SELECT total(length(sql)) FROM %s" },
   };
-  sqlite3_file *pFile = 0;
   int i;
   char *zSchemaTab;
   char *zDb = nArg>=2 ? azArg[1] : "main";
+  sqlite3_stmt *pStmt = 0;
   unsigned char aHdr[100];
   open_db(p, 0);
   if( p->db==0 ) return 1;
-  sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile);
-  if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){
-    return 1;
-  }
-  i = pFile->pMethods->xRead(pFile, aHdr, 100, 0);
-  if( i!=SQLITE_OK ){
+  sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1",
+                     -1, &pStmt, 0);
+  sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC);
+  if( sqlite3_step(pStmt)==SQLITE_ROW
+   && sqlite3_column_bytes(pStmt,0)>100
+  ){
+    memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100);
+    sqlite3_finalize(pStmt);
+  }else{
     raw_printf(stderr, "unable to read database header\n");
+    sqlite3_finalize(pStmt);
     return 1;
   }
   i = get2byteInt(aHdr+16);
diff --git a/src/shell.c.in b/src/shell.c.in
index db4f2e1128..54a61b9456 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3610,20 +3610,24 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
      { "schema size:",
        "SELECT total(length(sql)) FROM %s" },
   };
-  sqlite3_file *pFile = 0;
   int i;
   char *zSchemaTab;
   char *zDb = nArg>=2 ? azArg[1] : "main";
+  sqlite3_stmt *pStmt = 0;
   unsigned char aHdr[100];
   open_db(p, 0);
   if( p->db==0 ) return 1;
-  sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile);
-  if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){
-    return 1;
-  }
-  i = pFile->pMethods->xRead(pFile, aHdr, 100, 0);
-  if( i!=SQLITE_OK ){
+  sqlite3_prepare_v2(p->db,"SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1",
+                     -1, &pStmt, 0);
+  sqlite3_bind_text(pStmt, 1, zDb, -1, SQLITE_STATIC);
+  if( sqlite3_step(pStmt)==SQLITE_ROW
+   && sqlite3_column_bytes(pStmt,0)>100
+  ){
+    memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100);
+    sqlite3_finalize(pStmt);
+  }else{
     raw_printf(stderr, "unable to read database header\n");
+    sqlite3_finalize(pStmt);
     return 1;
   }
   i = get2byteInt(aHdr+16);

From 36187fe8c2b90068f3262f962654ba5632b530e7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Oct 2017 18:00:34 +0000
Subject: [PATCH 173/270] Add new extension "checkfreelist", which uses
 sqlite_dbpage to check that there are no invalid entries on the database
 free-list.

FossilOrigin-Name: 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4
---
 ext/misc/checkfreelist.c | 291 +++++++++++++++++++++++++++++++++++++++
 manifest                 |  14 +-
 manifest.uuid            |   2 +-
 test/checkfreelist.test  | 114 +++++++++++++++
 4 files changed, 414 insertions(+), 7 deletions(-)
 create mode 100644 ext/misc/checkfreelist.c
 create mode 100644 test/checkfreelist.test

diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c
new file mode 100644
index 0000000000..cec03b7999
--- /dev/null
+++ b/ext/misc/checkfreelist.c
@@ -0,0 +1,291 @@
+/*
+** 2017 October 11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This module exports a single C function:
+**
+**   int sqlite3_check_freelist(sqlite3 *db, const char *zDb);
+**
+** This function checks the free-list in database zDb (one of "main", 
+** "temp", etc.) and reports any errors by invoking the sqlite3_log()
+** function. It returns SQLITE_OK if successful, or an SQLite error
+** code otherwise. It is not an error if the free-list is corrupted but
+** no IO or OOM errors occur.
+**
+** If this file is compiled and loaded as an SQLite loadable extension,
+** it adds an SQL function "checkfreelist" to the database handle, to
+** be invoked as follows:
+**
+**   SELECT checkfreelist(<database-name>);
+**
+** This function performs the same checks as sqlite3_check_freelist(),
+** except that it returns all error messages as a single text value,
+** separated by newline characters. If the freelist is not corrupted
+** in any way, an empty string is returned.
+**
+** To compile this module for use as an SQLite loadable extension:
+**
+**   gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so
+*/
+
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+#ifndef SQLITE_AMALGAMATION
+# include <string.h>
+# include <stdio.h>
+# include <stdlib.h>
+# include <assert.h>
+# define ALWAYS(X)  1
+# define NEVER(X)   0
+  typedef unsigned char u8;
+  typedef unsigned short u16;
+  typedef unsigned int u32;
+#define get4byte(x) (        \
+    ((u32)((x)[0])<<24) +    \
+    ((u32)((x)[1])<<16) +    \
+    ((u32)((x)[2])<<8) +     \
+    ((u32)((x)[3]))          \
+)
+#endif
+
+/*
+** Execute a single PRAGMA statement and return the integer value returned
+** via output parameter (*pnOut).
+**
+** The SQL statement passed as the third argument should be a printf-style
+** format string containing a single "%s" which will be replace by the
+** value passed as the second argument. e.g.
+**
+**   sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut)
+**
+** executes "PRAGMA main.page_count" and stores the results in (*pnOut).
+*/
+static int sqlGetInteger(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Database name ("main", "temp" etc.) */
+  const char *zFmt,               /* SQL statement format */
+  u32 *pnOut                      /* OUT: Integer value */
+){
+  int rc, rc2;
+  char *zSql;
+  sqlite3_stmt *pStmt = 0;
+  int bOk = 0;
+
+  zSql = sqlite3_mprintf(zFmt, zDb);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+  }
+
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+    *pnOut = (u32)sqlite3_column_int(pStmt, 0);
+    bOk = 1;
+  }
+
+  rc2 = sqlite3_finalize(pStmt);
+  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR;
+  return rc;
+}
+
+/*
+** Argument zFmt must be a printf-style format string and must be 
+** followed by its required arguments. If argument pzOut is NULL, 
+** then the results of printf()ing the format string are passed to
+** sqlite3_log(). Otherwise, they are appended to the string
+** at (*pzOut).
+*/
+static int checkFreelistError(char **pzOut, const char *zFmt, ...){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  va_list ap;
+
+  va_start(ap, zFmt);
+  zErr = sqlite3_vmprintf(zFmt, ap);
+  if( zErr==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    if( pzOut ){
+      *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr);
+      if( *pzOut==0 ) rc = SQLITE_NOMEM;
+    }else{
+      sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr);
+    }
+    sqlite3_free(zErr);
+  }
+  va_end(ap);
+  return rc;
+}
+
+static int checkFreelist(
+  sqlite3 *db, 
+  const char *zDb,
+  char **pzOut
+){
+  /* This query returns one row for each page on the free list. Each row has
+  ** two columns - the page number and page content.  */
+  const char *zTrunk = 
+    "WITH freelist_trunk(i, d, n) AS ("
+      "SELECT 1, NULL, sqlite_readint32(data, 32) "
+      "FROM sqlite_dbpage(:1) WHERE pgno=1 "
+        "UNION ALL "
+      "SELECT n, data, sqlite_readint32(data) "
+      "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n "
+    ")"
+    "SELECT i, d FROM freelist_trunk WHERE i!=1;";
+
+  int rc, rc2;                    /* Return code */
+  sqlite3_stmt *pTrunk = 0;       /* Compilation of zTrunk */
+  u32 nPage = 0;                  /* Number of pages in db */
+  u32 nExpected = 0;              /* Expected number of free pages */
+  u32 nFree = 0;                  /* Number of pages on free list */
+
+  if( zDb==0 ) zDb = "main";
+
+  if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage))
+   || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected))
+  ){
+    return rc;
+  }
+
+  rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC);
+  while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){
+    u32 i;
+    u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0);
+    const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1);
+    int nData = sqlite3_column_bytes(pTrunk, 1);
+    u32 iNext = get4byte(&aData[0]);
+    u32 nLeaf = get4byte(&aData[4]);
+
+    nFree += 1+nLeaf;
+    if( iNext>nPage ){
+      rc = checkFreelistError(pzOut, 
+          "trunk page %d is out of range", (int)iNext
+      );
+    }
+
+    for(i=0; rc==SQLITE_OK && i<nLeaf; i++){
+      u32 iLeaf = get4byte(&aData[8 + 4*i]);
+      if( iLeaf==0 || iLeaf>nPage ){
+        rc = checkFreelistError(pzOut,
+            "leaf page %d is out of range (child %d of trunk page %d)", 
+            (int)iLeaf, (int)i, (int)iTrunk
+        );
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && nFree!=nExpected ){
+    rc = checkFreelistError(pzOut,
+        "free-list count mismatch: actual=%d header=%d", 
+        (int)nFree, (int)nExpected
+    );
+  }
+
+  rc2 = sqlite3_finalize(pTrunk);
+  if( rc==SQLITE_OK ) rc = rc2;
+  return rc;
+}
+
+int sqlite3_check_freelist(sqlite3 *db, const char *zDb){
+  return checkFreelist(db, zDb, 0);
+}
+
+static void checkfreelist_function(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  const char *zDb;
+  int rc;
+  char *zOut = 0;
+  sqlite3 *db = sqlite3_context_db_handle(pCtx);
+
+  assert( nArg==1 );
+  zDb = sqlite3_value_text(apArg[0]);
+  rc = checkFreelist(db, zDb, &zOut);
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+
+  sqlite3_free(zOut);
+}
+
+/*
+** An SQL function invoked as follows:
+**
+**   sqlite_readint32(BLOB)           -- Decode 32-bit integer from start of blob
+*/
+static void readint_function(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  const u8 *zBlob;
+  int nBlob;
+  int iOff = 0;
+  u32 iRet = 0;
+
+  if( nArg!=1 && nArg!=2 ){
+    sqlite3_result_error(
+        pCtx, "wrong number of arguments to function sqlite_readint32()", -1
+    );
+    return;
+  }
+  if( nArg==2 ){
+    iOff = sqlite3_value_int(apArg[1]);
+  }
+
+  zBlob = sqlite3_value_blob(apArg[0]);
+  nBlob = sqlite3_value_bytes(apArg[0]);
+
+  if( nBlob>=(iOff+4) ){
+    iRet = get4byte(&zBlob[iOff]);
+  }
+
+  sqlite3_result_int64(pCtx, (sqlite3_int64)iRet);
+}
+
+/*
+** Register the SQL functions.
+*/
+static int cflRegister(sqlite3 *db){
+  int rc = sqlite3_create_function(
+      db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0
+  );
+  if( rc!=SQLITE_OK ) return rc;
+  rc = sqlite3_create_function(
+      db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0
+  );
+  return rc;
+}
+
+/*
+** Extension load function.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_checkfreelist_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return cflRegister(db);
+}
diff --git a/manifest b/manifest
index 92b51678a2..d4e28e2d73 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable.
-D 2017-10-11T17:51:08.392
+C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list.
+D 2017-10-11T18:00:34.689
 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589
@@ -259,6 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
+F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
@@ -647,6 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe
 F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
+F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1658,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db
-R 32cd47bb05391f34b12cc1216f3e5e53
-U drh
-Z ef5ad8ed0ad3bdbe43b1de17eb74fe58
+P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388
+R 7024a507e1ac7d6985bebba168f4b31f
+U dan
+Z 54a66c878cf2157bd3590abc0ee2a612
diff --git a/manifest.uuid b/manifest.uuid
index 12bf79e9d8..e6ec495019 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388
\ No newline at end of file
+21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4
\ No newline at end of file
diff --git a/test/checkfreelist.test b/test/checkfreelist.test
new file mode 100644
index 0000000000..f16d8b57bb
--- /dev/null
+++ b/test/checkfreelist.test
@@ -0,0 +1,114 @@
+# 2017-10-11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the checkfreelist extension.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix checkfreelist
+
+ifcapable !vtab||!compound {
+  finish_test
+  return
+}
+
+if {[file exists ../checkfreelist.so]==0} {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+}
+
+db enable_load_extension 1
+do_execsql_test 1.1 {
+  SELECT load_extension('../checkfreelist.so');
+} {{}}
+
+do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok}
+do_execsql_test 1.3 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000
+  )
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+  DELETE FROM t1 WHERE rowid%3;
+  PRAGMA freelist_count;
+} {6726}
+
+do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok}
+do_execsql_test 1.5 {
+  WITH freelist_trunk(i, d, n) AS (
+    SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1
+      UNION ALL
+    SELECT n, data, sqlite_readint32(data) 
+    FROM freelist_trunk, sqlite_dbpage WHERE pgno=n
+  )
+  SELECT i FROM freelist_trunk WHERE i!=1;
+} {
+  10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235
+  4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5
+}
+
+do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok}
+
+proc set_int {blob idx newval} {
+  binary scan $blob I* ints
+  lset ints $idx $newval
+  binary format I* $ints
+}
+db func set_int set_int
+
+proc get_int {blob idx} {
+  binary scan $blob I* ints
+  lindex $ints $idx
+}
+db func get_int get_int
+
+do_execsql_test 1.7 {
+  BEGIN;
+    UPDATE sqlite_dbpage 
+      SET data = set_int(data, 1, get_int(data, 1)-1) 
+      WHERE pgno=4861;
+    SELECT checkfreelist('main');
+  ROLLBACK;
+} {{free-list count mismatch: actual=6725 header=6726}}
+
+do_execsql_test 1.8 {
+  BEGIN;
+    UPDATE sqlite_dbpage 
+      SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1)
+      WHERE pgno=4861;
+    SELECT checkfreelist('main');
+  ROLLBACK;
+} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}}
+
+do_execsql_test 1.9 {
+  BEGIN;
+    UPDATE sqlite_dbpage 
+      SET data = set_int(data, 5, 0)
+      WHERE pgno=4861;
+    SELECT checkfreelist('main');
+  ROLLBACK;
+} {{leaf page 0 is out of range (child 3 of trunk page 4861)}}
+
+do_execsql_test 1.10 {
+  BEGIN;
+    UPDATE sqlite_dbpage 
+      SET data = set_int(data, get_int(data, 1)+1, 0)
+      WHERE pgno=5;
+    SELECT checkfreelist('main');
+  ROLLBACK;
+} {{leaf page 0 is out of range (child 247 of trunk page 5)}}
+
+finish_test
+

From 7d157f91f4a29e8f237f1f1d0705271a3fe45f5f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Oct 2017 18:21:44 +0000
Subject: [PATCH 174/270] Check that the leaf count on each freelist trunk page
 is in range as part of checkfreelist processing.

FossilOrigin-Name: 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec
---
 ext/misc/checkfreelist.c |  8 ++++++++
 manifest                 | 14 +++++++-------
 manifest.uuid            |  2 +-
 test/checkfreelist.test  |  9 +++++++++
 4 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c
index cec03b7999..345f26c223 100644
--- a/ext/misc/checkfreelist.c
+++ b/ext/misc/checkfreelist.c
@@ -170,6 +170,14 @@ static int checkFreelist(
     u32 iNext = get4byte(&aData[0]);
     u32 nLeaf = get4byte(&aData[4]);
 
+    if( nLeaf>((nData/4)-2-6) ){
+      rc = checkFreelistError(pzOut, 
+          "leaf count out of range (%d) on trunk page %d", 
+          (int)nLeaf, (int)iTrunk
+      );
+      nLeaf = (nData/4) - 2 - 6;
+    }
+
     nFree += 1+nLeaf;
     if( iNext>nPage ){
       rc = checkFreelistError(pzOut, 
diff --git a/manifest b/manifest
index d4e28e2d73..7a6b18c230 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list.
-D 2017-10-11T18:00:34.689
+C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing.
+D 2017-10-11T18:21:44.429
 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589
@@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
-F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c
+F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
@@ -648,7 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe
 F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
-F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f
+F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388
-R 7024a507e1ac7d6985bebba168f4b31f
+P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4
+R f25a07255c10ac9143e97a5d9d8e737e
 U dan
-Z 54a66c878cf2157bd3590abc0ee2a612
+Z fe2b858df1a95da0fbc85fd6ca1ee6ea
diff --git a/manifest.uuid b/manifest.uuid
index e6ec495019..a9df98f06f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4
\ No newline at end of file
+4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec
\ No newline at end of file
diff --git a/test/checkfreelist.test b/test/checkfreelist.test
index f16d8b57bb..93e4ecc234 100644
--- a/test/checkfreelist.test
+++ b/test/checkfreelist.test
@@ -110,5 +110,14 @@ do_execsql_test 1.10 {
   ROLLBACK;
 } {{leaf page 0 is out of range (child 247 of trunk page 5)}}
 
+do_execsql_test 1.11 {
+  BEGIN;
+    UPDATE sqlite_dbpage 
+      SET data = set_int(data, 1, 249)
+      WHERE pgno=5;
+    SELECT checkfreelist('main');
+  ROLLBACK;
+} {{leaf count out of range (249) on trunk page 5}}
+
 finish_test
 

From f294ce648b121fc5ed1c8e6456ed6b03fac34b13 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 11 Oct 2017 18:26:26 +0000
Subject: [PATCH 175/270] Add the checkfreelist extension to the command-line
 shell.

FossilOrigin-Name: 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52
---
 ext/misc/checkfreelist.c |   2 +-
 manifest                 |  18 +--
 manifest.uuid            |   2 +-
 src/shell.c              | 302 +++++++++++++++++++++++++++++++++++++++
 src/shell.c.in           |   2 +
 5 files changed, 315 insertions(+), 11 deletions(-)

diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c
index 345f26c223..cd2801e040 100644
--- a/ext/misc/checkfreelist.c
+++ b/ext/misc/checkfreelist.c
@@ -223,7 +223,7 @@ static void checkfreelist_function(
   sqlite3 *db = sqlite3_context_db_handle(pCtx);
 
   assert( nArg==1 );
-  zDb = sqlite3_value_text(apArg[0]);
+  zDb = (const char*)sqlite3_value_text(apArg[0]);
   rc = checkFreelist(db, zDb, &zOut);
   if( rc==SQLITE_OK ){
     sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT);
diff --git a/manifest b/manifest
index 7a6b18c230..c2dd3cb1eb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Check\sthat\sthe\sleaf\scount\son\seach\sfreelist\strunk\spage\sis\sin\srange\sas\spart\sof\ncheckfreelist\sprocessing.
-D 2017-10-11T18:21:44.429
+C Add\sthe\scheckfreelist\sextension\sto\sthe\scommand-line\sshell.
+D 2017-10-11T18:26:26.636
 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589
@@ -259,7 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
-F ext/misc/checkfreelist.c fc46557e73a6233bd698815d3963acc44bf4dba0ca9c91c90be361cca49d6b3e
+F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
@@ -461,8 +461,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e
-F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18
+F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57
+F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1660,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4
-R f25a07255c10ac9143e97a5d9d8e737e
-U dan
-Z fe2b858df1a95da0fbc85fd6ca1ee6ea
+P 4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec
+R 0bf65201c138febf2eab9baca075a19d
+U drh
+Z f892c9c71f9811efc31d97f9328db29a
diff --git a/manifest.uuid b/manifest.uuid
index a9df98f06f..eb5db27581 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4e89406248f51d3b83d61e5472fb493f3d3b4ff2a69bf256c7e15445eeb2f3ec
\ No newline at end of file
+48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index 2b77d482e6..6c14db9fdd 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -2156,6 +2156,307 @@ int sqlite3_completion_init(
 }
 
 /************************* End ../ext/misc/completion.c ********************/
+/************************* Begin ../ext/misc/checkfreelist.c ******************/
+/*
+** 2017 October 11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This module exports a single C function:
+**
+**   int sqlite3_check_freelist(sqlite3 *db, const char *zDb);
+**
+** This function checks the free-list in database zDb (one of "main", 
+** "temp", etc.) and reports any errors by invoking the sqlite3_log()
+** function. It returns SQLITE_OK if successful, or an SQLite error
+** code otherwise. It is not an error if the free-list is corrupted but
+** no IO or OOM errors occur.
+**
+** If this file is compiled and loaded as an SQLite loadable extension,
+** it adds an SQL function "checkfreelist" to the database handle, to
+** be invoked as follows:
+**
+**   SELECT checkfreelist(<database-name>);
+**
+** This function performs the same checks as sqlite3_check_freelist(),
+** except that it returns all error messages as a single text value,
+** separated by newline characters. If the freelist is not corrupted
+** in any way, an empty string is returned.
+**
+** To compile this module for use as an SQLite loadable extension:
+**
+**   gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so
+*/
+
+SQLITE_EXTENSION_INIT1
+
+#ifndef SQLITE_AMALGAMATION
+# include <string.h>
+# include <stdio.h>
+# include <stdlib.h>
+# include <assert.h>
+# define ALWAYS(X)  1
+# define NEVER(X)   0
+  typedef unsigned char u8;
+  typedef unsigned short u16;
+  typedef unsigned int u32;
+#define get4byte(x) (        \
+    ((u32)((x)[0])<<24) +    \
+    ((u32)((x)[1])<<16) +    \
+    ((u32)((x)[2])<<8) +     \
+    ((u32)((x)[3]))          \
+)
+#endif
+
+/*
+** Execute a single PRAGMA statement and return the integer value returned
+** via output parameter (*pnOut).
+**
+** The SQL statement passed as the third argument should be a printf-style
+** format string containing a single "%s" which will be replace by the
+** value passed as the second argument. e.g.
+**
+**   sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut)
+**
+** executes "PRAGMA main.page_count" and stores the results in (*pnOut).
+*/
+static int sqlGetInteger(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Database name ("main", "temp" etc.) */
+  const char *zFmt,               /* SQL statement format */
+  u32 *pnOut                      /* OUT: Integer value */
+){
+  int rc, rc2;
+  char *zSql;
+  sqlite3_stmt *pStmt = 0;
+  int bOk = 0;
+
+  zSql = sqlite3_mprintf(zFmt, zDb);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+  }
+
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+    *pnOut = (u32)sqlite3_column_int(pStmt, 0);
+    bOk = 1;
+  }
+
+  rc2 = sqlite3_finalize(pStmt);
+  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR;
+  return rc;
+}
+
+/*
+** Argument zFmt must be a printf-style format string and must be 
+** followed by its required arguments. If argument pzOut is NULL, 
+** then the results of printf()ing the format string are passed to
+** sqlite3_log(). Otherwise, they are appended to the string
+** at (*pzOut).
+*/
+static int checkFreelistError(char **pzOut, const char *zFmt, ...){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  va_list ap;
+
+  va_start(ap, zFmt);
+  zErr = sqlite3_vmprintf(zFmt, ap);
+  if( zErr==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    if( pzOut ){
+      *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr);
+      if( *pzOut==0 ) rc = SQLITE_NOMEM;
+    }else{
+      sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr);
+    }
+    sqlite3_free(zErr);
+  }
+  va_end(ap);
+  return rc;
+}
+
+static int checkFreelist(
+  sqlite3 *db, 
+  const char *zDb,
+  char **pzOut
+){
+  /* This query returns one row for each page on the free list. Each row has
+  ** two columns - the page number and page content.  */
+  const char *zTrunk = 
+    "WITH freelist_trunk(i, d, n) AS ("
+      "SELECT 1, NULL, sqlite_readint32(data, 32) "
+      "FROM sqlite_dbpage(:1) WHERE pgno=1 "
+        "UNION ALL "
+      "SELECT n, data, sqlite_readint32(data) "
+      "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n "
+    ")"
+    "SELECT i, d FROM freelist_trunk WHERE i!=1;";
+
+  int rc, rc2;                    /* Return code */
+  sqlite3_stmt *pTrunk = 0;       /* Compilation of zTrunk */
+  u32 nPage = 0;                  /* Number of pages in db */
+  u32 nExpected = 0;              /* Expected number of free pages */
+  u32 nFree = 0;                  /* Number of pages on free list */
+
+  if( zDb==0 ) zDb = "main";
+
+  if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage))
+   || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected))
+  ){
+    return rc;
+  }
+
+  rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC);
+  while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){
+    u32 i;
+    u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0);
+    const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1);
+    int nData = sqlite3_column_bytes(pTrunk, 1);
+    u32 iNext = get4byte(&aData[0]);
+    u32 nLeaf = get4byte(&aData[4]);
+
+    if( nLeaf>((nData/4)-2-6) ){
+      rc = checkFreelistError(pzOut, 
+          "leaf count out of range (%d) on trunk page %d", 
+          (int)nLeaf, (int)iTrunk
+      );
+      nLeaf = (nData/4) - 2 - 6;
+    }
+
+    nFree += 1+nLeaf;
+    if( iNext>nPage ){
+      rc = checkFreelistError(pzOut, 
+          "trunk page %d is out of range", (int)iNext
+      );
+    }
+
+    for(i=0; rc==SQLITE_OK && i<nLeaf; i++){
+      u32 iLeaf = get4byte(&aData[8 + 4*i]);
+      if( iLeaf==0 || iLeaf>nPage ){
+        rc = checkFreelistError(pzOut,
+            "leaf page %d is out of range (child %d of trunk page %d)", 
+            (int)iLeaf, (int)i, (int)iTrunk
+        );
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && nFree!=nExpected ){
+    rc = checkFreelistError(pzOut,
+        "free-list count mismatch: actual=%d header=%d", 
+        (int)nFree, (int)nExpected
+    );
+  }
+
+  rc2 = sqlite3_finalize(pTrunk);
+  if( rc==SQLITE_OK ) rc = rc2;
+  return rc;
+}
+
+int sqlite3_check_freelist(sqlite3 *db, const char *zDb){
+  return checkFreelist(db, zDb, 0);
+}
+
+static void checkfreelist_function(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  const char *zDb;
+  int rc;
+  char *zOut = 0;
+  sqlite3 *db = sqlite3_context_db_handle(pCtx);
+
+  assert( nArg==1 );
+  zDb = (const char*)sqlite3_value_text(apArg[0]);
+  rc = checkFreelist(db, zDb, &zOut);
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+
+  sqlite3_free(zOut);
+}
+
+/*
+** An SQL function invoked as follows:
+**
+**   sqlite_readint32(BLOB)           -- Decode 32-bit integer from start of blob
+*/
+static void readint_function(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  const u8 *zBlob;
+  int nBlob;
+  int iOff = 0;
+  u32 iRet = 0;
+
+  if( nArg!=1 && nArg!=2 ){
+    sqlite3_result_error(
+        pCtx, "wrong number of arguments to function sqlite_readint32()", -1
+    );
+    return;
+  }
+  if( nArg==2 ){
+    iOff = sqlite3_value_int(apArg[1]);
+  }
+
+  zBlob = sqlite3_value_blob(apArg[0]);
+  nBlob = sqlite3_value_bytes(apArg[0]);
+
+  if( nBlob>=(iOff+4) ){
+    iRet = get4byte(&zBlob[iOff]);
+  }
+
+  sqlite3_result_int64(pCtx, (sqlite3_int64)iRet);
+}
+
+/*
+** Register the SQL functions.
+*/
+static int cflRegister(sqlite3 *db){
+  int rc = sqlite3_create_function(
+      db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0
+  );
+  if( rc!=SQLITE_OK ) return rc;
+  rc = sqlite3_create_function(
+      db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0
+  );
+  return rc;
+}
+
+/*
+** Extension load function.
+*/
+#ifdef _WIN32
+
+#endif
+int sqlite3_checkfreelist_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return cflRegister(db);
+}
+
+/************************* End ../ext/misc/checkfreelist.c ********************/
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -4246,6 +4547,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
+    sqlite3_checkfreelist_init(p->db, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }
diff --git a/src/shell.c.in b/src/shell.c.in
index 54a61b9456..de170887f7 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -796,6 +796,7 @@ static void shellAddSchemaName(
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+INCLUDE ../ext/misc/checkfreelist.c
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -2886,6 +2887,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
+    sqlite3_checkfreelist_init(p->db, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }

From 3c48e6551b7577c3ae9fa3c78c5497bbdb7d2609 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Oct 2017 20:27:03 +0000
Subject: [PATCH 176/270] Add test file mmapwarm.test. With tests for the
 sqlite3_mmap_warm() extension.

FossilOrigin-Name: 5c4980ef17291dcea5e93ead353b9a95e2fbff56ff7257e9878d095f6827b4e9
---
 manifest           | 13 ++++----
 manifest.uuid      |  2 +-
 test/mmapwarm.test | 80 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+), 7 deletions(-)
 create mode 100644 test/mmapwarm.test

diff --git a/manifest b/manifest
index b8e6ca811e..d3e7c578b6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sspeed-check.sh\stest\sscript,\sallow\san\sadditional\stest-name\sargument\nwhich\sbecomes\sthe\scomparison\sbaseline,\sin\splace\sof\s"trunk".
-D 2017-10-11T12:20:36.082
+C Add\stest\sfile\smmapwarm.test.\sWith\stests\sfor\sthe\ssqlite3_mmap_warm()\sextension.
+D 2017-10-11T20:27:03.845
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -1054,6 +1054,7 @@ F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
 F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93
 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3
+F test/mmapwarm.test fea1c17c538afaf4cdc914ad2a3f9c9c34a5d00e9a5d6423ddb6083c91aed6ab
 F test/multiplex.test dc0d67b66f84b484a83cb8bbdf3f0a7f49562ccd
 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a
 F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101
@@ -1656,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b8c1b5a584aeb6ab63cff875ca16135efeadac16b4b32afa589845477feebf32
-R 9a03aa36b167c86705816bfaa31cdc19
-U drh
-Z 8f5e2f1e221681e7d4bf20eb32b86754
+P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
+R a1117ec420d64b4ef2d3498dc99fc25e
+U dan
+Z 24c1dfd62587ab303e50a574dce24d2b
diff --git a/manifest.uuid b/manifest.uuid
index 7305d2515a..c160449091 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
\ No newline at end of file
+5c4980ef17291dcea5e93ead353b9a95e2fbff56ff7257e9878d095f6827b4e9
\ No newline at end of file
diff --git a/test/mmapwarm.test b/test/mmapwarm.test
new file mode 100644
index 0000000000..6072ab8624
--- /dev/null
+++ b/test/mmapwarm.test
@@ -0,0 +1,80 @@
+# 20 September 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+
+if 0 {
+  db close
+  sqlite3_shutdown
+  proc msg {args} { puts $args }
+  test_sqlite3_log msg
+  sqlite3 db test.db
+}
+
+set testprefix mmapwarm
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x, y);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+  )
+  INSERT INTO t1 SELECT randomblob(400), randomblob(500) FROM s;
+  PRAGMA page_count;
+} {507}
+db close
+
+do_test 1.1 {
+  sqlite3 db test.db
+  db eval {PRAGMA mmap_size = 1000000}
+  sqlite3_mmap_warm db
+} {SQLITE_OK}
+
+do_test 1.2 {
+  db close
+  sqlite3 db test.db
+  db eval {PRAGMA mmap_size = 1000000}
+  sqlite3_mmap_warm db "main"
+} {SQLITE_OK}
+
+do_test 1.3 {
+  sqlite3 db test.db
+  sqlite3_mmap_warm db
+} {SQLITE_OK}
+
+do_test 1.4 {
+  db close
+  sqlite3 db test.db
+  sqlite3_mmap_warm db "main"
+} {SQLITE_OK}
+
+do_test 2.0 {
+  db close
+  sqlite3 db test.db
+  db eval BEGIN
+  sqlite3_mmap_warm db "main"
+} {SQLITE_MISUSE}
+
+do_faultsim_test 3 -faults oom* -prep {
+  sqlite3 db test.db
+  sqlite3_db_config_lookaside db 0 0 0
+  db eval { PRAGMA mmap_size = 1000000 }
+  db eval { SELECT * FROM sqlite_master }
+} -body {
+  sqlite3_mmap_warm db "main"
+} -test {
+  faultsim_test_result {0 SQLITE_OK} {0 SQLITE_NOMEM}
+}
+ 
+finish_test

From 8a18100d4409d1d7bd91e5d44a55b9ceb52ef707 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 01:19:06 +0000
Subject: [PATCH 177/270] Fix a typo in a numeric constant in an assert() - a
 bug that has been present in the code since check-in [79e22b95038] on
 2010-03-30.

FossilOrigin-Name: f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/btree.c   |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 804cc5e872..ba7a68f797 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sJSON1\sdefines\sconsistent\sin\sthe\smakefiles\sfor\sMSVC.
-D 2017-10-11T17:50:22.538
+C Fix\sa\stypo\sin\sa\snumeric\sconstant\sin\san\sassert()\s-\sa\sbug\sthat\shas\sbeen\spresent\nin\sthe\scode\ssince\scheck-in\s[79e22b95038]\son\s2010-03-30.
+D 2017-10-12T01:19:06.385
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 307701b46e4ac0e2aaa7776ea5936fff21636e991c9d5988584f37d65be9c13e
@@ -401,7 +401,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 07ad643c75519359f72622bfb862e53723e0bed52ef7c9979e04a0a531078e34
+F src/btree.c 8565b061a6a6fad850230c73d6a7a8ffb88f3370e3352a8689a9a672160c5cc5
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 79c96121baf065fd78ba4f1fe82ce725b6372df20e9323d88022b7c243701372
-R 2d35bd3f9bb9d088160d8caa2fcda76b
-U mistachkin
-Z ae3424fdef9b7840b735184e2f02e403
+P 31eee0e4bffc5419e076a589049bfea7327dfc666fcc2767c0b700f506055c4e
+R 53da87efcb541e81caf23f4e430fb62a
+U drh
+Z 3fb702a07bd6ea398ba326fd478602ce
diff --git a/manifest.uuid b/manifest.uuid
index f9b265f962..e291c426f6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-31eee0e4bffc5419e076a589049bfea7327dfc666fcc2767c0b700f506055c4e
\ No newline at end of file
+f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 1496588f85..89222197bc 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -2040,7 +2040,7 @@ static Pgno btreePagecount(BtShared *pBt){
 }
 u32 sqlite3BtreeLastPage(Btree *p){
   assert( sqlite3BtreeHoldsMutex(p) );
-  assert( ((p->pBt->nPage)&0x8000000)==0 );
+  assert( ((p->pBt->nPage)&0x80000000)==0 );
   return btreePagecount(p->pBt);
 }
 

From 4c6cddcaabb6a5539cd87347e6be96fe8d1d2d37 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 10:28:30 +0000
Subject: [PATCH 178/270] Make sure the tableColumnList() routine of the
 command-line shell does not cause a null-pointer dereference in an error
 condition.

FossilOrigin-Name: 5d0ceb8dcdef92cd72307e532a4a6c269b2c458fecb0bbede0bb941099eebc5b
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c    |  1 +
 src/shell.c.in |  1 +
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index ba7a68f797..8625e176b3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stypo\sin\sa\snumeric\sconstant\sin\san\sassert()\s-\sa\sbug\sthat\shas\sbeen\spresent\nin\sthe\scode\ssince\scheck-in\s[79e22b95038]\son\s2010-03-30.
-D 2017-10-12T01:19:06.385
+C Make\ssure\sthe\stableColumnList()\sroutine\sof\sthe\scommand-line\sshell\sdoes\snot\ncause\sa\snull-pointer\sdereference\sin\san\serror\scondition.
+D 2017-10-12T10:28:30.906
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 307701b46e4ac0e2aaa7776ea5936fff21636e991c9d5988584f37d65be9c13e
@@ -459,8 +459,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe
-F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0
+F src/shell.c b7a097667fbf48f5b45ffdd789dcfa2c7bd8985ebb284aedddaf6d1d1101237f
+F src/shell.c.in 423944f4ad73a7e73d9c06e645e19ac1aa5f45c22069936e3a008b28a5df8003
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 31eee0e4bffc5419e076a589049bfea7327dfc666fcc2767c0b700f506055c4e
-R 53da87efcb541e81caf23f4e430fb62a
+P f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b
+R d1ecfc96a3abe2209d6b1155d4df5e91
 U drh
-Z 3fb702a07bd6ea398ba326fd478602ce
+Z 7fd7ee0ab2a7b6f6de891ca66c37b5d6
diff --git a/manifest.uuid b/manifest.uuid
index e291c426f6..e2ecde5c63 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b
\ No newline at end of file
+5d0ceb8dcdef92cd72307e532a4a6c269b2c458fecb0bbede0bb941099eebc5b
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index f7be941570..1537d64a06 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -3801,6 +3801,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
     }
   }
   sqlite3_finalize(pStmt);
+  if( azCol==0 ) return 0;
   azCol[0] = 0;
   azCol[nCol+1] = 0;
 
diff --git a/src/shell.c.in b/src/shell.c.in
index db4f2e1128..896d475ed2 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2441,6 +2441,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
     }
   }
   sqlite3_finalize(pStmt);
+  if( azCol==0 ) return 0;
   azCol[0] = 0;
   azCol[nCol+1] = 0;
 

From e1b972bd38071d3b78bf34a08e27db490e5fd6d6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 12 Oct 2017 11:13:34 +0000
Subject: [PATCH 179/270] Update some test cases to work with
 SQLITE_OMIT_VIRTUALTABLE and DEFAULT_AUTOVACUUM builds.

FossilOrigin-Name: 4b68f424ca9b5d4b0e77065a4e60ec0f70dcce0d01053fd7a6b2ecae9d3a5967
---
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 test/bigmmap.test  |  2 +-
 test/mmapwarm.test |  1 +
 test/whereF.test   |  2 +-
 5 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index d3e7c578b6..48a0614763 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\sfile\smmapwarm.test.\sWith\stests\sfor\sthe\ssqlite3_mmap_warm()\sextension.
-D 2017-10-11T20:27:03.845
+C Update\ssome\stest\scases\sto\swork\swith\sSQLITE_OMIT_VIRTUALTABLE\sand\nDEFAULT_AUTOVACUUM\sbuilds.
+D 2017-10-12T11:13:34.180
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -615,7 +615,7 @@ F test/bestindex5.test 412b42f8036b28d8b2f3534d89389ad946a4b1a65a12263f51936f742
 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c
 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59
 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc
-F test/bigmmap.test ed6058a7794be26865c94d5bb62e12cdc4f7f01562b3b04f13eb3cdc52783921
+F test/bigmmap.test abe819e6e1ac1db0c3bfe364ff58889d96e7896b2bbc8bdf1afc77cdeb7d7a9b
 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747
 F test/bigsort.test 8299fa9298f4f1e02fc7d2712e8b77d6cd60e5a2
 F test/bind.test 1e136709b306f7ed3192d349c2930d89df6ab621654ad6f1a72381d3fe76f483
@@ -1054,7 +1054,7 @@ F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
 F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93
 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3
-F test/mmapwarm.test fea1c17c538afaf4cdc914ad2a3f9c9c34a5d00e9a5d6423ddb6083c91aed6ab
+F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3
 F test/multiplex.test dc0d67b66f84b484a83cb8bbdf3f0a7f49562ccd
 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a
 F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101
@@ -1536,7 +1536,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test a44c7d73c22e2532a5c2dc7377fca64825f86a8969b113a1bdcffacbcbc14d39
+F test/whereF.test d44b58338fe5ddd7286023e9bedb255aa264a6c4d2168b49591b167371c675c7
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0245adffc6f9b580217e0d2feb396d6895e54cdc25f5dfc9c8f4090b919e9e49
-R a1117ec420d64b4ef2d3498dc99fc25e
+P 5c4980ef17291dcea5e93ead353b9a95e2fbff56ff7257e9878d095f6827b4e9
+R b29f3e00752cf4173db0ec3aec606d66
 U dan
-Z 24c1dfd62587ab303e50a574dce24d2b
+Z bf0ac220dc2474d5d13a0301e1eb0593
diff --git a/manifest.uuid b/manifest.uuid
index c160449091..ba180b21c1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c4980ef17291dcea5e93ead353b9a95e2fbff56ff7257e9878d095f6827b4e9
\ No newline at end of file
+4b68f424ca9b5d4b0e77065a4e60ec0f70dcce0d01053fd7a6b2ecae9d3a5967
\ No newline at end of file
diff --git a/test/bigmmap.test b/test/bigmmap.test
index d44c487fc2..9284bda37d 100644
--- a/test/bigmmap.test
+++ b/test/bigmmap.test
@@ -20,7 +20,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix bigmmap
 
-ifcapable !mmap {
+ifcapable !mmap||!vtab {
   finish_test
   return
 }
diff --git a/test/mmapwarm.test b/test/mmapwarm.test
index 6072ab8624..b077047ebc 100644
--- a/test/mmapwarm.test
+++ b/test/mmapwarm.test
@@ -26,6 +26,7 @@ set testprefix mmapwarm
 
 
 do_execsql_test 1.0 {
+  PRAGMA auto_vacuum = 0;
   CREATE TABLE t1(x, y);
   WITH s(i) AS (
     SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
diff --git a/test/whereF.test b/test/whereF.test
index 0888f25789..8b272a5c53 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -179,7 +179,7 @@ do_test 5.6 { expr [db status vmstep]<200 } 1
 # 2017-09-04 ticket b899b6042f97f52d
 # Segfault on correlated subquery...
 #
-ifcapable json1 {
+ifcapable json1&&vtab {
   do_execsql_test 6.1 {
     CREATE TABLE t6(x);
     SELECT * FROM t6 WHERE 1 IN (SELECT value FROM json_each(x));

From 122ab8fd9ddc54be1243033ce1b7462b1597927d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 12 Oct 2017 11:33:13 +0000
Subject: [PATCH 180/270] Fix a potential crash that may follow an OOM or other
 system error in the shell tool.

FossilOrigin-Name: c36761e1a05c15c48468dd793faf06636f5dcfd3b4314074c4d9c9eac5c2695c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/shell.c   |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 48a0614763..03107352b2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\ssome\stest\scases\sto\swork\swith\sSQLITE_OMIT_VIRTUALTABLE\sand\nDEFAULT_AUTOVACUUM\sbuilds.
-D 2017-10-12T11:13:34.180
+C Fix\sa\spotential\scrash\sthat\smay\sfollow\san\sOOM\sor\sother\ssystem\serror\sin\sthe\sshell\stool.
+D 2017-10-12T11:33:13.900
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -459,7 +459,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c cbf450e75665a185c546adc702ec5fd091306ae7a08bc88b1508ac9c11acc7fe
+F src/shell.c edadbfd5b21ec8fadf8844f087b2c1224cb9db57614b4a69c33e732436296f43
 F src/shell.c.in e03f7d473e10b65c25836a058a3e7a1665ffb1fe712949dcd6e38c790e4eafd0
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5c4980ef17291dcea5e93ead353b9a95e2fbff56ff7257e9878d095f6827b4e9
-R b29f3e00752cf4173db0ec3aec606d66
+P 4b68f424ca9b5d4b0e77065a4e60ec0f70dcce0d01053fd7a6b2ecae9d3a5967
+R 83b72d43ce085ee1e083f5ceb39f739a
 U dan
-Z bf0ac220dc2474d5d13a0301e1eb0593
+Z e19953ef492c0be13869a8f2e3e20ba9
diff --git a/manifest.uuid b/manifest.uuid
index ba180b21c1..2c35c8915a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4b68f424ca9b5d4b0e77065a4e60ec0f70dcce0d01053fd7a6b2ecae9d3a5967
\ No newline at end of file
+c36761e1a05c15c48468dd793faf06636f5dcfd3b4314074c4d9c9eac5c2695c
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index f7be941570..06ffb32b8c 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -3801,6 +3801,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
     }
   }
   sqlite3_finalize(pStmt);
+  if( !azCol ) return 0;
   azCol[0] = 0;
   azCol[nCol+1] = 0;
 

From aa62d2e48fe554bddeccab8837f344452137ded4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 13:47:48 +0000
Subject: [PATCH 181/270] The src/shell.c file is now generated from
 src/shell.c.in, so remove shell.c from version control and update the
 makefiles to build it automatically.

FossilOrigin-Name: 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956
---
 Makefile.in       |   18 +-
 Makefile.msc      |   22 +-
 main.mk           |   17 +-
 manifest          |   19 +-
 manifest.uuid     |    2 +-
 src/shell.c       | 8378 ---------------------------------------------
 tool/mkshellc.tcl |   11 +-
 7 files changed, 60 insertions(+), 8407 deletions(-)
 delete mode 100644 src/shell.c

diff --git a/Makefile.in b/Makefile.in
index 58c2c2919b..be032c07b5 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -265,7 +265,7 @@ SRC = \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
+  $(TOP)/src/shell.c.in \
   $(TOP)/src/sqlite.h.in \
   $(TOP)/src/sqlite3ext.h \
   $(TOP)/src/sqliteInt.h \
@@ -362,6 +362,7 @@ SRC += \
   parse.c \
   parse.h \
   config.h \
+  shell.c \
   sqlite3.h
 
 # Source code to the test files.
@@ -597,9 +598,9 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-version-info "8:6:8" \
 		-avoid-version
 
-sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
+sqlite3$(TEXE):	shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
-		$(TOP)/src/shell.c sqlite3.c \
+		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
 
 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
@@ -971,6 +972,17 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/fileio.c \
+	$(TOP)/ext/misc/completion.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
+
+
 
 
 # Rules to build the extension objects.
diff --git a/Makefile.msc b/Makefile.msc
index 0fed23dce6..4c3c67f94d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1220,11 +1220,6 @@ SRC01 = \
   $(TOP)\src\wherecode.c \
   $(TOP)\src\whereexpr.c
 
-# Shell source code files.
-#
-SRC02 = \
-  $(TOP)\src\shell.c
-
 # Core miscellaneous files.
 #
 SRC03 = \
@@ -1331,6 +1326,7 @@ SRC11 = \
   keywordhash.h \
   opcodes.h \
   parse.h \
+  shell.c \
   $(SQLITE3H)
 
 # Generated Tcl header files
@@ -1345,7 +1341,7 @@ SRC12 =
 
 # All source code files.
 #
-SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
+SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
 
 # Source code to the test files.
 #
@@ -1569,8 +1565,8 @@ sqlite3.def:	libsqlite3.lib
 		| sort >> sqlite3.def
 # <</block2>>
 
-$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 # <<mark>>
@@ -1629,7 +1625,6 @@ mptest:	mptester.exe
 	-mkdir tsrc
 	for %i in ($(SRC00)) do copy /Y %i tsrc
 	for %i in ($(SRC01)) do copy /Y %i tsrc
-	for %i in ($(SRC02)) do copy /Y %i tsrc
 	for %i in ($(SRC03)) do copy /Y %i tsrc
 	for %i in ($(SRC04)) do copy /Y %i tsrc
 	for %i in ($(SRC05)) do copy /Y %i tsrc
@@ -1974,6 +1969,15 @@ mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\misc\shathree.c \
+	$(TOP)\ext\misc\fileio.c \
+	$(TOP)\ext\misc\completion.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c
 
 
 # Rules to build the extension objects.
diff --git a/main.mk b/main.mk
index 7da6db15f1..a4da2a37d0 100644
--- a/main.mk
+++ b/main.mk
@@ -146,7 +146,7 @@ SRC = \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \
-  $(TOP)/src/shell.c \
+  $(TOP)/src/shell.c.in \
   $(TOP)/src/sqlite.h.in \
   $(TOP)/src/sqlite3ext.h \
   $(TOP)/src/sqliteInt.h \
@@ -270,6 +270,7 @@ SRC += \
   opcodes.h \
   parse.c \
   parse.h \
+  shell.c \
   sqlite3.h
 
 
@@ -496,9 +497,9 @@ libsqlite3.a:	$(LIBOBJ)
 	$(AR) libsqlite3.a $(LIBOBJ)
 	$(RANLIB) libsqlite3.a
 
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
+sqlite3$(EXE):	shell.c libsqlite3.a sqlite3.h
 	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
-		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
+		shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
 
 sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
@@ -658,6 +659,16 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash >keywordhash.h
 
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/fileio.c \
+	$(TOP)/ext/misc/completion.c
+
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
+	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
+
 
 
 # Rules to build the extension objects.
diff --git a/manifest b/manifest
index f8b45db8f8..eadda83d65 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Merge\sfixes\sfrom\sthe\s3.21\sbranch.
-D 2017-10-12T13:21:56.153
-F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
+C The\ssrc/shell.c\sfile\sis\snow\sgenerated\sfrom\ssrc/shell.c.in,\sso\sremove\sshell.c\nfrom\sversion\scontrol\sand\supdate\sthe\smakefiles\sto\sbuild\sit\sautomatically.
+D 2017-10-12T13:47:48.544
+F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 307701b46e4ac0e2aaa7776ea5936fff21636e991c9d5988584f37d65be9c13e
+F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -382,7 +382,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk d0145f02deb67d65c4822225847cba112c237cdb62f4905eeb4b648e82bfc222
+F main.mk 3e671408634fb8e8eaa296e80627066a2524053db5a9c5c28c6ec06cf7e99a51
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -459,7 +459,6 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c b7a097667fbf48f5b45ffdd789dcfa2c7bd8985ebb284aedddaf6d1d1101237f
 F src/shell.c.in 423944f4ad73a7e73d9c06e645e19ac1aa5f45c22069936e3a008b28a5df8003
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1594,7 +1593,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 950c36f45941c12140e346a907fb66198bc2770ff7a17c749201e78d34bb3b0b
+F tool/mkshellc.tcl 574307265b49d813301fba91ccd74e6a26d33f65f74b6891c320a0ffbee07895
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1657,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c36761e1a05c15c48468dd793faf06636f5dcfd3b4314074c4d9c9eac5c2695c 5d0ceb8dcdef92cd72307e532a4a6c269b2c458fecb0bbede0bb941099eebc5b
-R 5e22761bfa955ea525bc4446de4276dc
+P 292921692c8919d29f0a67d03ca953d5c1c4900d8c8567cceab27513732be598
+R 55fc95c237c6d381b00d7c404f68ae59
 U drh
-Z 1f315bd34e44a8389c3ed773831f4232
+Z 4cb9f1b7e5b96c77468dac8db9ac0cf0
diff --git a/manifest.uuid b/manifest.uuid
index f3e08d8726..b48ef93c79 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-292921692c8919d29f0a67d03ca953d5c1c4900d8c8567cceab27513732be598
\ No newline at end of file
+36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
deleted file mode 100644
index 1537d64a06..0000000000
--- a/src/shell.c
+++ /dev/null
@@ -1,8378 +0,0 @@
-/* DO NOT EDIT!
-** This file is automatically generated by the script in the canonical
-** SQLite source tree at tool/mkshellc.tcl.  That script combines source
-** code from various constituent source files of SQLite into this single
-** "shell.c" file used to implement the SQLite command-line shell.
-**
-** Most of the code found below comes from the "src/shell.c.in" file in
-** the canonical SQLite source tree.  That main file contains "INCLUDE"
-** lines that specify other files in the canonical source tree that are
-** inserted to getnerate this complete program source file.
-**
-** The code from multiple files is combined into this single "shell.c"
-** source file to help make the command-line program easier to compile.
-**
-** To modify this program, get a copy of the canonical SQLite source tree,
-** edit the src/shell.c.in" and/or some of the other files that are included
-** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script.
-*/
-/*
-** 2001 September 15
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file contains code to implement the "sqlite" command line
-** utility for accessing SQLite databases.
-*/
-#if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS)
-/* This needs to come before any includes for MSVC compiler */
-#define _CRT_SECURE_NO_WARNINGS
-#endif
-
-/*
-** Warning pragmas copied from msvc.h in the core.
-*/
-#if defined(_MSC_VER)
-#pragma warning(disable : 4054)
-#pragma warning(disable : 4055)
-#pragma warning(disable : 4100)
-#pragma warning(disable : 4127)
-#pragma warning(disable : 4130)
-#pragma warning(disable : 4152)
-#pragma warning(disable : 4189)
-#pragma warning(disable : 4206)
-#pragma warning(disable : 4210)
-#pragma warning(disable : 4232)
-#pragma warning(disable : 4244)
-#pragma warning(disable : 4305)
-#pragma warning(disable : 4306)
-#pragma warning(disable : 4702)
-#pragma warning(disable : 4706)
-#endif /* defined(_MSC_VER) */
-
-/*
-** No support for loadable extensions in VxWorks.
-*/
-#if (defined(__RTP__) || defined(_WRS_KERNEL)) && !SQLITE_OMIT_LOAD_EXTENSION
-# define SQLITE_OMIT_LOAD_EXTENSION 1
-#endif
-
-/*
-** Enable large-file support for fopen() and friends on unix.
-*/
-#ifndef SQLITE_DISABLE_LFS
-# define _LARGE_FILE       1
-# ifndef _FILE_OFFSET_BITS
-#   define _FILE_OFFSET_BITS 64
-# endif
-# define _LARGEFILE_SOURCE 1
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <assert.h>
-#include "sqlite3.h"
-#if SQLITE_USER_AUTHENTICATION
-# include "sqlite3userauth.h"
-#endif
-#include <ctype.h>
-#include <stdarg.h>
-
-#if !defined(_WIN32) && !defined(WIN32)
-# include <signal.h>
-# if !defined(__RTP__) && !defined(_WRS_KERNEL)
-#  include <pwd.h>
-# endif
-# include <unistd.h>
-# include <sys/types.h>
-#endif
-
-#if HAVE_READLINE
-# include <readline/readline.h>
-# include <readline/history.h>
-#endif
-
-#if HAVE_EDITLINE
-# include <editline/readline.h>
-#endif
-
-#if HAVE_EDITLINE || HAVE_READLINE
-
-# define shell_add_history(X) add_history(X)
-# define shell_read_history(X) read_history(X)
-# define shell_write_history(X) write_history(X)
-# define shell_stifle_history(X) stifle_history(X)
-# define shell_readline(X) readline(X)
-
-#elif HAVE_LINENOISE
-
-# include "linenoise.h"
-# define shell_add_history(X) linenoiseHistoryAdd(X)
-# define shell_read_history(X) linenoiseHistoryLoad(X)
-# define shell_write_history(X) linenoiseHistorySave(X)
-# define shell_stifle_history(X) linenoiseHistorySetMaxLen(X)
-# define shell_readline(X) linenoise(X)
-
-#else
-
-# define shell_read_history(X)
-# define shell_write_history(X)
-# define shell_stifle_history(X)
-
-# define SHELL_USE_LOCAL_GETLINE 1
-#endif
-
-
-#if defined(_WIN32) || defined(WIN32)
-# include <io.h>
-# include <fcntl.h>
-# define isatty(h) _isatty(h)
-# ifndef access
-#  define access(f,m) _access((f),(m))
-# endif
-# undef popen
-# define popen _popen
-# undef pclose
-# define pclose _pclose
-#else
- /* Make sure isatty() has a prototype. */
- extern int isatty(int);
-
-# if !defined(__RTP__) && !defined(_WRS_KERNEL)
-  /* popen and pclose are not C89 functions and so are
-  ** sometimes omitted from the <stdio.h> header */
-   extern FILE *popen(const char*,const char*);
-   extern int pclose(FILE*);
-# else
-#  define SQLITE_OMIT_POPEN 1
-# endif
-#endif
-
-#if defined(_WIN32_WCE)
-/* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty()
- * thus we always assume that we have a console. That can be
- * overridden with the -batch command line option.
- */
-#define isatty(x) 1
-#endif
-
-/* ctype macros that work with signed characters */
-#define IsSpace(X)  isspace((unsigned char)X)
-#define IsDigit(X)  isdigit((unsigned char)X)
-#define ToLower(X)  (char)tolower((unsigned char)X)
-
-#if defined(_WIN32) || defined(WIN32)
-#include <windows.h>
-
-/* string conversion routines only needed on Win32 */
-extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR);
-extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int);
-extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int);
-extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText);
-#endif
-
-/* On Windows, we normally run with output mode of TEXT so that \n characters
-** are automatically translated into \r\n.  However, this behavior needs
-** to be disabled in some cases (ex: when generating CSV output and when
-** rendering quoted strings that contain \n characters).  The following
-** routines take care of that.
-*/
-#if defined(_WIN32) || defined(WIN32)
-static void setBinaryMode(FILE *file, int isOutput){
-  if( isOutput ) fflush(file);
-  _setmode(_fileno(file), _O_BINARY);
-}
-static void setTextMode(FILE *file, int isOutput){
-  if( isOutput ) fflush(file);
-  _setmode(_fileno(file), _O_TEXT);
-}
-#else
-# define setBinaryMode(X,Y)
-# define setTextMode(X,Y)
-#endif
-
-
-/* True if the timer is enabled */
-static int enableTimer = 0;
-
-/* Return the current wall-clock time */
-static sqlite3_int64 timeOfDay(void){
-  static sqlite3_vfs *clockVfs = 0;
-  sqlite3_int64 t;
-  if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
-  if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
-    clockVfs->xCurrentTimeInt64(clockVfs, &t);
-  }else{
-    double r;
-    clockVfs->xCurrentTime(clockVfs, &r);
-    t = (sqlite3_int64)(r*86400000.0);
-  }
-  return t;
-}
-
-#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux)
-#include <sys/time.h>
-#include <sys/resource.h>
-
-/* VxWorks does not support getrusage() as far as we can determine */
-#if defined(_WRS_KERNEL) || defined(__RTP__)
-struct rusage {
-  struct timeval ru_utime; /* user CPU time used */
-  struct timeval ru_stime; /* system CPU time used */
-};
-#define getrusage(A,B) memset(B,0,sizeof(*B))
-#endif
-
-/* Saved resource information for the beginning of an operation */
-static struct rusage sBegin;  /* CPU time at start */
-static sqlite3_int64 iBegin;  /* Wall-clock time at start */
-
-/*
-** Begin timing an operation
-*/
-static void beginTimer(void){
-  if( enableTimer ){
-    getrusage(RUSAGE_SELF, &sBegin);
-    iBegin = timeOfDay();
-  }
-}
-
-/* Return the difference of two time_structs in seconds */
-static double timeDiff(struct timeval *pStart, struct timeval *pEnd){
-  return (pEnd->tv_usec - pStart->tv_usec)*0.000001 +
-         (double)(pEnd->tv_sec - pStart->tv_sec);
-}
-
-/*
-** Print the timing results.
-*/
-static void endTimer(void){
-  if( enableTimer ){
-    sqlite3_int64 iEnd = timeOfDay();
-    struct rusage sEnd;
-    getrusage(RUSAGE_SELF, &sEnd);
-    printf("Run Time: real %.3f user %f sys %f\n",
-       (iEnd - iBegin)*0.001,
-       timeDiff(&sBegin.ru_utime, &sEnd.ru_utime),
-       timeDiff(&sBegin.ru_stime, &sEnd.ru_stime));
-  }
-}
-
-#define BEGIN_TIMER beginTimer()
-#define END_TIMER endTimer()
-#define HAS_TIMER 1
-
-#elif (defined(_WIN32) || defined(WIN32))
-
-/* Saved resource information for the beginning of an operation */
-static HANDLE hProcess;
-static FILETIME ftKernelBegin;
-static FILETIME ftUserBegin;
-static sqlite3_int64 ftWallBegin;
-typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME,
-                                    LPFILETIME, LPFILETIME);
-static GETPROCTIMES getProcessTimesAddr = NULL;
-
-/*
-** Check to see if we have timer support.  Return 1 if necessary
-** support found (or found previously).
-*/
-static int hasTimer(void){
-  if( getProcessTimesAddr ){
-    return 1;
-  } else {
-    /* GetProcessTimes() isn't supported in WIN95 and some other Windows
-    ** versions. See if the version we are running on has it, and if it
-    ** does, save off a pointer to it and the current process handle.
-    */
-    hProcess = GetCurrentProcess();
-    if( hProcess ){
-      HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll"));
-      if( NULL != hinstLib ){
-        getProcessTimesAddr =
-            (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes");
-        if( NULL != getProcessTimesAddr ){
-          return 1;
-        }
-        FreeLibrary(hinstLib);
-      }
-    }
-  }
-  return 0;
-}
-
-/*
-** Begin timing an operation
-*/
-static void beginTimer(void){
-  if( enableTimer && getProcessTimesAddr ){
-    FILETIME ftCreation, ftExit;
-    getProcessTimesAddr(hProcess,&ftCreation,&ftExit,
-                        &ftKernelBegin,&ftUserBegin);
-    ftWallBegin = timeOfDay();
-  }
-}
-
-/* Return the difference of two FILETIME structs in seconds */
-static double timeDiff(FILETIME *pStart, FILETIME *pEnd){
-  sqlite_int64 i64Start = *((sqlite_int64 *) pStart);
-  sqlite_int64 i64End = *((sqlite_int64 *) pEnd);
-  return (double) ((i64End - i64Start) / 10000000.0);
-}
-
-/*
-** Print the timing results.
-*/
-static void endTimer(void){
-  if( enableTimer && getProcessTimesAddr){
-    FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd;
-    sqlite3_int64 ftWallEnd = timeOfDay();
-    getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd);
-    printf("Run Time: real %.3f user %f sys %f\n",
-       (ftWallEnd - ftWallBegin)*0.001,
-       timeDiff(&ftUserBegin, &ftUserEnd),
-       timeDiff(&ftKernelBegin, &ftKernelEnd));
-  }
-}
-
-#define BEGIN_TIMER beginTimer()
-#define END_TIMER endTimer()
-#define HAS_TIMER hasTimer()
-
-#else
-#define BEGIN_TIMER
-#define END_TIMER
-#define HAS_TIMER 0
-#endif
-
-/*
-** Used to prevent warnings about unused parameters
-*/
-#define UNUSED_PARAMETER(x) (void)(x)
-
-/*
-** If the following flag is set, then command execution stops
-** at an error if we are not interactive.
-*/
-static int bail_on_error = 0;
-
-/*
-** Threat stdin as an interactive input if the following variable
-** is true.  Otherwise, assume stdin is connected to a file or pipe.
-*/
-static int stdin_is_interactive = 1;
-
-/*
-** On Windows systems we have to know if standard output is a console
-** in order to translate UTF-8 into MBCS.  The following variable is
-** true if translation is required.
-*/
-static int stdout_is_console = 1;
-
-/*
-** The following is the open SQLite database.  We make a pointer
-** to this database a static variable so that it can be accessed
-** by the SIGINT handler to interrupt database processing.
-*/
-static sqlite3 *globalDb = 0;
-
-/*
-** True if an interrupt (Control-C) has been received.
-*/
-static volatile int seenInterrupt = 0;
-
-/*
-** This is the name of our program. It is set in main(), used
-** in a number of other places, mostly for error messages.
-*/
-static char *Argv0;
-
-/*
-** Prompt strings. Initialized in main. Settable with
-**   .prompt main continue
-*/
-static char mainPrompt[20];     /* First line prompt. default: "sqlite> "*/
-static char continuePrompt[20]; /* Continuation prompt. default: "   ...> " */
-
-/*
-** Render output like fprintf().  Except, if the output is going to the
-** console and if this is running on a Windows machine, translate the
-** output from UTF-8 into MBCS.
-*/
-#if defined(_WIN32) || defined(WIN32)
-void utf8_printf(FILE *out, const char *zFormat, ...){
-  va_list ap;
-  va_start(ap, zFormat);
-  if( stdout_is_console && (out==stdout || out==stderr) ){
-    char *z1 = sqlite3_vmprintf(zFormat, ap);
-    char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0);
-    sqlite3_free(z1);
-    fputs(z2, out);
-    sqlite3_free(z2);
-  }else{
-    vfprintf(out, zFormat, ap);
-  }
-  va_end(ap);
-}
-#elif !defined(utf8_printf)
-# define utf8_printf fprintf
-#endif
-
-/*
-** Render output like fprintf().  This should not be used on anything that
-** includes string formatting (e.g. "%s").
-*/
-#if !defined(raw_printf)
-# define raw_printf fprintf
-#endif
-
-/*
-** Write I/O traces to the following stream.
-*/
-#ifdef SQLITE_ENABLE_IOTRACE
-static FILE *iotrace = 0;
-#endif
-
-/*
-** This routine works like printf in that its first argument is a
-** format string and subsequent arguments are values to be substituted
-** in place of % fields.  The result of formatting this string
-** is written to iotrace.
-*/
-#ifdef SQLITE_ENABLE_IOTRACE
-static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){
-  va_list ap;
-  char *z;
-  if( iotrace==0 ) return;
-  va_start(ap, zFormat);
-  z = sqlite3_vmprintf(zFormat, ap);
-  va_end(ap);
-  utf8_printf(iotrace, "%s", z);
-  sqlite3_free(z);
-}
-#endif
-
-/*
-** Output string zUtf to stream pOut as w characters.  If w is negative,
-** then right-justify the text.  W is the width in UTF-8 characters, not
-** in bytes.  This is different from the %*.*s specification in printf
-** since with %*.*s the width is measured in bytes, not characters.
-*/
-static void utf8_width_print(FILE *pOut, int w, const char *zUtf){
-  int i;
-  int n;
-  int aw = w<0 ? -w : w;
-  char zBuf[1000];
-  if( aw>(int)sizeof(zBuf)/3 ) aw = (int)sizeof(zBuf)/3;
-  for(i=n=0; zUtf[i]; i++){
-    if( (zUtf[i]&0xc0)!=0x80 ){
-      n++;
-      if( n==aw ){
-        do{ i++; }while( (zUtf[i]&0xc0)==0x80 );
-        break;
-      }
-    }
-  }
-  if( n>=aw ){
-    utf8_printf(pOut, "%.*s", i, zUtf);
-  }else if( w<0 ){
-    utf8_printf(pOut, "%*s%s", aw-n, "", zUtf);
-  }else{
-    utf8_printf(pOut, "%s%*s", zUtf, aw-n, "");
-  }
-}
-
-
-/*
-** Determines if a string is a number of not.
-*/
-static int isNumber(const char *z, int *realnum){
-  if( *z=='-' || *z=='+' ) z++;
-  if( !IsDigit(*z) ){
-    return 0;
-  }
-  z++;
-  if( realnum ) *realnum = 0;
-  while( IsDigit(*z) ){ z++; }
-  if( *z=='.' ){
-    z++;
-    if( !IsDigit(*z) ) return 0;
-    while( IsDigit(*z) ){ z++; }
-    if( realnum ) *realnum = 1;
-  }
-  if( *z=='e' || *z=='E' ){
-    z++;
-    if( *z=='+' || *z=='-' ) z++;
-    if( !IsDigit(*z) ) return 0;
-    while( IsDigit(*z) ){ z++; }
-    if( realnum ) *realnum = 1;
-  }
-  return *z==0;
-}
-
-/*
-** Compute a string length that is limited to what can be stored in
-** lower 30 bits of a 32-bit signed integer.
-*/
-static int strlen30(const char *z){
-  const char *z2 = z;
-  while( *z2 ){ z2++; }
-  return 0x3fffffff & (int)(z2 - z);
-}
-
-/*
-** Return the length of a string in characters.  Multibyte UTF8 characters
-** count as a single character.
-*/
-static int strlenChar(const char *z){
-  int n = 0;
-  while( *z ){
-    if( (0xc0&*(z++))!=0x80 ) n++;
-  }
-  return n;
-}
-
-/*
-** This routine reads a line of text from FILE in, stores
-** the text in memory obtained from malloc() and returns a pointer
-** to the text.  NULL is returned at end of file, or if malloc()
-** fails.
-**
-** If zLine is not NULL then it is a malloced buffer returned from
-** a previous call to this routine that may be reused.
-*/
-static char *local_getline(char *zLine, FILE *in){
-  int nLine = zLine==0 ? 0 : 100;
-  int n = 0;
-
-  while( 1 ){
-    if( n+100>nLine ){
-      nLine = nLine*2 + 100;
-      zLine = realloc(zLine, nLine);
-      if( zLine==0 ) return 0;
-    }
-    if( fgets(&zLine[n], nLine - n, in)==0 ){
-      if( n==0 ){
-        free(zLine);
-        return 0;
-      }
-      zLine[n] = 0;
-      break;
-    }
-    while( zLine[n] ) n++;
-    if( n>0 && zLine[n-1]=='\n' ){
-      n--;
-      if( n>0 && zLine[n-1]=='\r' ) n--;
-      zLine[n] = 0;
-      break;
-    }
-  }
-#if defined(_WIN32) || defined(WIN32)
-  /* For interactive input on Windows systems, translate the
-  ** multi-byte characterset characters into UTF-8. */
-  if( stdin_is_interactive && in==stdin ){
-    char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
-    if( zTrans ){
-      int nTrans = strlen30(zTrans)+1;
-      if( nTrans>nLine ){
-        zLine = realloc(zLine, nTrans);
-        if( zLine==0 ){
-          sqlite3_free(zTrans);
-          return 0;
-        }
-      }
-      memcpy(zLine, zTrans, nTrans);
-      sqlite3_free(zTrans);
-    }
-  }
-#endif /* defined(_WIN32) || defined(WIN32) */
-  return zLine;
-}
-
-/*
-** Retrieve a single line of input text.
-**
-** If in==0 then read from standard input and prompt before each line.
-** If isContinuation is true, then a continuation prompt is appropriate.
-** If isContinuation is zero, then the main prompt should be used.
-**
-** If zPrior is not NULL then it is a buffer from a prior call to this
-** routine that can be reused.
-**
-** The result is stored in space obtained from malloc() and must either
-** be freed by the caller or else passed back into this routine via the
-** zPrior argument for reuse.
-*/
-static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
-  char *zPrompt;
-  char *zResult;
-  if( in!=0 ){
-    zResult = local_getline(zPrior, in);
-  }else{
-    zPrompt = isContinuation ? continuePrompt : mainPrompt;
-#if SHELL_USE_LOCAL_GETLINE
-    printf("%s", zPrompt);
-    fflush(stdout);
-    zResult = local_getline(zPrior, stdin);
-#else
-    free(zPrior);
-    zResult = shell_readline(zPrompt);
-    if( zResult && *zResult ) shell_add_history(zResult);
-#endif
-  }
-  return zResult;
-}
-/*
-** A variable length string to which one can append text.
-*/
-typedef struct ShellText ShellText;
-struct ShellText {
-  char *z;
-  int n;
-  int nAlloc;
-};
-
-/*
-** Initialize and destroy a ShellText object
-*/
-static void initText(ShellText *p){
-  memset(p, 0, sizeof(*p));
-}
-static void freeText(ShellText *p){
-  free(p->z);
-  initText(p);
-}
-
-/* zIn is either a pointer to a NULL-terminated string in memory obtained
-** from malloc(), or a NULL pointer. The string pointed to by zAppend is
-** added to zIn, and the result returned in memory obtained from malloc().
-** zIn, if it was not NULL, is freed.
-**
-** If the third argument, quote, is not '\0', then it is used as a
-** quote character for zAppend.
-*/
-static void appendText(ShellText *p, char const *zAppend, char quote){
-  int len;
-  int i;
-  int nAppend = strlen30(zAppend);
-
-  len = nAppend+p->n+1;
-  if( quote ){
-    len += 2;
-    for(i=0; i<nAppend; i++){
-      if( zAppend[i]==quote ) len++;
-    }
-  }
-
-  if( p->n+len>=p->nAlloc ){
-    p->nAlloc = p->nAlloc*2 + len + 20;
-    p->z = realloc(p->z, p->nAlloc);
-    if( p->z==0 ){
-      memset(p, 0, sizeof(*p));
-      return;
-    }
-  }
-
-  if( quote ){
-    char *zCsr = p->z+p->n;
-    *zCsr++ = quote;
-    for(i=0; i<nAppend; i++){
-      *zCsr++ = zAppend[i];
-      if( zAppend[i]==quote ) *zCsr++ = quote;
-    }
-    *zCsr++ = quote;
-    p->n = (int)(zCsr - p->z);
-    *zCsr = '\0';
-  }else{
-    memcpy(p->z+p->n, zAppend, nAppend);
-    p->n += nAppend;
-    p->z[p->n] = '\0';
-  }
-}
-
-/*
-** Attempt to determine if identifier zName needs to be quoted, either
-** because it contains non-alphanumeric characters, or because it is an
-** SQLite keyword.  Be conservative in this estimate:  When in doubt assume
-** that quoting is required.
-**
-** Return '"' if quoting is required.  Return 0 if no quoting is required.
-*/
-static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
-  if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
-  for(i=0; zName[i]; i++){
-    if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
-  }
-  lwr = 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
-}
-
-/*
-** SQL function:  shell_add_schema(S,X)
-**
-** Add the schema name X to the CREATE statement in S and return the result.
-** Examples:
-**
-**    CREATE TABLE t1(x)   ->   CREATE TABLE xyz.t1(x);
-**
-** Also works on
-**
-**    CREATE INDEX
-**    CREATE UNIQUE INDEX
-**    CREATE VIEW
-**    CREATE TRIGGER
-**    CREATE VIRTUAL TABLE
-**
-** This UDF is used by the .schema command to insert the schema name of
-** attached databases into the middle of the sqlite_master.sql field.
-*/
-static void shellAddSchemaName(
-  sqlite3_context *pCtx,
-  int nVal,
-  sqlite3_value **apVal
-){
-  static const char *aPrefix[] = {
-     "TABLE",
-     "INDEX",
-     "UNIQUE INDEX",
-     "VIEW",
-     "TRIGGER",
-     "VIRTUAL TABLE"
-  };
-  int i = 0;
-  const char *zIn = (const char*)sqlite3_value_text(apVal[0]);
-  const char *zSchema = (const char*)sqlite3_value_text(apVal[1]);
-  assert( nVal==2 );
-  if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
-    for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){
-      int n = strlen30(aPrefix[i]);
-      if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){
-        char cQuote = quoteChar(zSchema);
-        char *z;
-        if( cQuote ){
-         z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8);
-        }else{
-          z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8);
-        }
-        sqlite3_result_text(pCtx, z, -1, sqlite3_free);
-        return;
-      }
-    }
-  }
-  sqlite3_result_value(pCtx, apVal[0]);
-}
-
-/*
-** The source code for several run-time loadable extensions is inserted
-** below by the ../tool/mkshellc.tcl script.  Before processing that included
-** code, we need to override some macros to make the included program code
-** work here in the middle of this regular program.
-*/
-#define SQLITE_EXTENSION_INIT1
-#define SQLITE_EXTENSION_INIT2(X) (void)(X)
-
-/************************* Begin ../ext/misc/shathree.c ******************/
-/*
-** 2017-03-08
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This SQLite extension implements a functions that compute SHA1 hashes.
-** Two SQL functions are implemented:
-**
-**     sha3(X,SIZE)
-**     sha3_query(Y,SIZE)
-**
-** The sha3(X) function computes the SHA3 hash of the input X, or NULL if
-** X is NULL.
-**
-** The sha3_query(Y) function evalutes all queries in the SQL statements of Y
-** and returns a hash of their results.
-**
-** The SIZE argument is optional.  If omitted, the SHA3-256 hash algorithm
-** is used.  If SIZE is included it must be one of the integers 224, 256,
-** 384, or 512, to determine SHA3 hash variant that is computed.
-*/
-SQLITE_EXTENSION_INIT1
-#include <assert.h>
-#include <string.h>
-#include <stdarg.h>
-typedef sqlite3_uint64 u64;
-
-/******************************************************************************
-** The Hash Engine
-*/
-/*
-** Macros to determine whether the machine is big or little endian,
-** and whether or not that determination is run-time or compile-time.
-**
-** For best performance, an attempt is made to guess at the byte-order
-** using C-preprocessor macros.  If that is unsuccessful, or if
-** -DSHA3_BYTEORDER=0 is set, then byte-order is determined
-** at run-time.
-*/
-#ifndef SHA3_BYTEORDER
-# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
-     defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
-     defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
-     defined(__arm__)
-#   define SHA3_BYTEORDER    1234
-# elif defined(sparc)    || defined(__ppc__)
-#   define SHA3_BYTEORDER    4321
-# else
-#   define SHA3_BYTEORDER 0
-# endif
-#endif
-
-
-/*
-** State structure for a SHA3 hash in progress
-*/
-typedef struct SHA3Context SHA3Context;
-struct SHA3Context {
-  union {
-    u64 s[25];                /* Keccak state. 5x5 lines of 64 bits each */
-    unsigned char x[1600];    /* ... or 1600 bytes */
-  } u;
-  unsigned nRate;        /* Bytes of input accepted per Keccak iteration */
-  unsigned nLoaded;      /* Input bytes loaded into u.x[] so far this cycle */
-  unsigned ixMask;       /* Insert next input into u.x[nLoaded^ixMask]. */
-};
-
-/*
-** A single step of the Keccak mixing function for a 1600-bit state
-*/
-static void KeccakF1600Step(SHA3Context *p){
-  int i;
-  u64 B0, B1, B2, B3, B4;
-  u64 C0, C1, C2, C3, C4;
-  u64 D0, D1, D2, D3, D4;
-  static const u64 RC[] = {
-    0x0000000000000001ULL,  0x0000000000008082ULL,
-    0x800000000000808aULL,  0x8000000080008000ULL,
-    0x000000000000808bULL,  0x0000000080000001ULL,
-    0x8000000080008081ULL,  0x8000000000008009ULL,
-    0x000000000000008aULL,  0x0000000000000088ULL,
-    0x0000000080008009ULL,  0x000000008000000aULL,
-    0x000000008000808bULL,  0x800000000000008bULL,
-    0x8000000000008089ULL,  0x8000000000008003ULL,
-    0x8000000000008002ULL,  0x8000000000000080ULL,
-    0x000000000000800aULL,  0x800000008000000aULL,
-    0x8000000080008081ULL,  0x8000000000008080ULL,
-    0x0000000080000001ULL,  0x8000000080008008ULL
-  };
-# define A00 (p->u.s[0])
-# define A01 (p->u.s[1])
-# define A02 (p->u.s[2])
-# define A03 (p->u.s[3])
-# define A04 (p->u.s[4])
-# define A10 (p->u.s[5])
-# define A11 (p->u.s[6])
-# define A12 (p->u.s[7])
-# define A13 (p->u.s[8])
-# define A14 (p->u.s[9])
-# define A20 (p->u.s[10])
-# define A21 (p->u.s[11])
-# define A22 (p->u.s[12])
-# define A23 (p->u.s[13])
-# define A24 (p->u.s[14])
-# define A30 (p->u.s[15])
-# define A31 (p->u.s[16])
-# define A32 (p->u.s[17])
-# define A33 (p->u.s[18])
-# define A34 (p->u.s[19])
-# define A40 (p->u.s[20])
-# define A41 (p->u.s[21])
-# define A42 (p->u.s[22])
-# define A43 (p->u.s[23])
-# define A44 (p->u.s[24])
-# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
-
-  for(i=0; i<24; i+=4){
-    C0 = A00^A10^A20^A30^A40;
-    C1 = A01^A11^A21^A31^A41;
-    C2 = A02^A12^A22^A32^A42;
-    C3 = A03^A13^A23^A33^A43;
-    C4 = A04^A14^A24^A34^A44;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
-
-    B0 = (A00^D0);
-    B1 = ROL64((A11^D1), 44);
-    B2 = ROL64((A22^D2), 43);
-    B3 = ROL64((A33^D3), 21);
-    B4 = ROL64((A44^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i];
-    A11 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
-
-    B2 = ROL64((A20^D0), 3);
-    B3 = ROL64((A31^D1), 45);
-    B4 = ROL64((A42^D2), 61);
-    B0 = ROL64((A03^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A20 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
-
-    B4 = ROL64((A40^D0), 18);
-    B0 = ROL64((A01^D1), 1);
-    B1 = ROL64((A12^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A34^D4), 8);
-    A40 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
-
-    B1 = ROL64((A10^D0), 36);
-    B2 = ROL64((A21^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A43^D3), 56);
-    B0 = ROL64((A04^D4), 27);
-    A10 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
-
-    B3 = ROL64((A30^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A02^D2), 62);
-    B1 = ROL64((A13^D3), 55);
-    B2 = ROL64((A24^D4), 39);
-    A30 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
-
-    C0 = A00^A20^A40^A10^A30;
-    C1 = A11^A31^A01^A21^A41;
-    C2 = A22^A42^A12^A32^A02;
-    C3 = A33^A03^A23^A43^A13;
-    C4 = A44^A14^A34^A04^A24;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
-
-    B0 = (A00^D0);
-    B1 = ROL64((A31^D1), 44);
-    B2 = ROL64((A12^D2), 43);
-    B3 = ROL64((A43^D3), 21);
-    B4 = ROL64((A24^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+1];
-    A31 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
-
-    B2 = ROL64((A40^D0), 3);
-    B3 = ROL64((A21^D1), 45);
-    B4 = ROL64((A02^D2), 61);
-    B0 = ROL64((A33^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A40 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
-
-    B4 = ROL64((A30^D0), 18);
-    B0 = ROL64((A11^D1), 1);
-    B1 = ROL64((A42^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A04^D4), 8);
-    A30 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
-
-    B1 = ROL64((A20^D0), 36);
-    B2 = ROL64((A01^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A13^D3), 56);
-    B0 = ROL64((A44^D4), 27);
-    A20 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
-
-    B3 = ROL64((A10^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A22^D2), 62);
-    B1 = ROL64((A03^D3), 55);
-    B2 = ROL64((A34^D4), 39);
-    A10 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
-
-    C0 = A00^A40^A30^A20^A10;
-    C1 = A31^A21^A11^A01^A41;
-    C2 = A12^A02^A42^A32^A22;
-    C3 = A43^A33^A23^A13^A03;
-    C4 = A24^A14^A04^A44^A34;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
-
-    B0 = (A00^D0);
-    B1 = ROL64((A21^D1), 44);
-    B2 = ROL64((A42^D2), 43);
-    B3 = ROL64((A13^D3), 21);
-    B4 = ROL64((A34^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+2];
-    A21 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
-
-    B2 = ROL64((A30^D0), 3);
-    B3 = ROL64((A01^D1), 45);
-    B4 = ROL64((A22^D2), 61);
-    B0 = ROL64((A43^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A30 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
-
-    B4 = ROL64((A10^D0), 18);
-    B0 = ROL64((A31^D1), 1);
-    B1 = ROL64((A02^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A44^D4), 8);
-    A10 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
-
-    B1 = ROL64((A40^D0), 36);
-    B2 = ROL64((A11^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A03^D3), 56);
-    B0 = ROL64((A24^D4), 27);
-    A40 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
-
-    B3 = ROL64((A20^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A12^D2), 62);
-    B1 = ROL64((A33^D3), 55);
-    B2 = ROL64((A04^D4), 39);
-    A20 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
-
-    C0 = A00^A30^A10^A40^A20;
-    C1 = A21^A01^A31^A11^A41;
-    C2 = A42^A22^A02^A32^A12;
-    C3 = A13^A43^A23^A03^A33;
-    C4 = A34^A14^A44^A24^A04;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
-
-    B0 = (A00^D0);
-    B1 = ROL64((A01^D1), 44);
-    B2 = ROL64((A02^D2), 43);
-    B3 = ROL64((A03^D3), 21);
-    B4 = ROL64((A04^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+3];
-    A01 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
-
-    B2 = ROL64((A10^D0), 3);
-    B3 = ROL64((A11^D1), 45);
-    B4 = ROL64((A12^D2), 61);
-    B0 = ROL64((A13^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A10 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
-
-    B4 = ROL64((A20^D0), 18);
-    B0 = ROL64((A21^D1), 1);
-    B1 = ROL64((A22^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A24^D4), 8);
-    A20 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
-
-    B1 = ROL64((A30^D0), 36);
-    B2 = ROL64((A31^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A33^D3), 56);
-    B0 = ROL64((A34^D4), 27);
-    A30 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
-
-    B3 = ROL64((A40^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A42^D2), 62);
-    B1 = ROL64((A43^D3), 55);
-    B2 = ROL64((A44^D4), 39);
-    A40 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
-  }
-}
-
-/*
-** Initialize a new hash.  iSize determines the size of the hash
-** in bits and should be one of 224, 256, 384, or 512.  Or iSize
-** can be zero to use the default hash size of 256 bits.
-*/
-static void SHA3Init(SHA3Context *p, int iSize){
-  memset(p, 0, sizeof(*p));
-  if( iSize>=128 && iSize<=512 ){
-    p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
-  }else{
-    p->nRate = (1600 - 2*256)/8;
-  }
-#if SHA3_BYTEORDER==1234
-  /* Known to be little-endian at compile-time. No-op */
-#elif SHA3_BYTEORDER==4321
-  p->ixMask = 7;  /* Big-endian */
-#else
-  {
-    static unsigned int one = 1;
-    if( 1==*(unsigned char*)&one ){
-      /* Little endian.  No byte swapping. */
-      p->ixMask = 0;
-    }else{
-      /* Big endian.  Byte swap. */
-      p->ixMask = 7;
-    }
-  }
-#endif
-}
-
-/*
-** Make consecutive calls to the SHA3Update function to add new content
-** to the hash
-*/
-static void SHA3Update(
-  SHA3Context *p,
-  const unsigned char *aData,
-  unsigned int nData
-){
-  unsigned int i = 0;
-#if SHA3_BYTEORDER==1234
-  if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
-    for(; i+7<nData; i+=8){
-      p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
-      p->nLoaded += 8;
-      if( p->nLoaded>=p->nRate ){
-        KeccakF1600Step(p);
-        p->nLoaded = 0;
-      }
-    }
-  }
-#endif
-  for(; i<nData; i++){
-#if SHA3_BYTEORDER==1234
-    p->u.x[p->nLoaded] ^= aData[i];
-#elif SHA3_BYTEORDER==4321
-    p->u.x[p->nLoaded^0x07] ^= aData[i];
-#else
-    p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
-#endif
-    p->nLoaded++;
-    if( p->nLoaded==p->nRate ){
-      KeccakF1600Step(p);
-      p->nLoaded = 0;
-    }
-  }
-}
-
-/*
-** After all content has been added, invoke SHA3Final() to compute
-** the final hash.  The function returns a pointer to the binary
-** hash value.
-*/
-static unsigned char *SHA3Final(SHA3Context *p){
-  unsigned int i;
-  if( p->nLoaded==p->nRate-1 ){
-    const unsigned char c1 = 0x86;
-    SHA3Update(p, &c1, 1);
-  }else{
-    const unsigned char c2 = 0x06;
-    const unsigned char c3 = 0x80;
-    SHA3Update(p, &c2, 1);
-    p->nLoaded = p->nRate - 1;
-    SHA3Update(p, &c3, 1);
-  }
-  for(i=0; i<p->nRate; i++){
-    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
-  }
-  return &p->u.x[p->nRate];
-}
-/* End of the hashing logic
-*****************************************************************************/
-
-/*
-** Implementation of the sha3(X,SIZE) function.
-**
-** Return a BLOB which is the SIZE-bit SHA3 hash of X.  The default
-** size is 256.  If X is a BLOB, it is hashed as is.  
-** For all other non-NULL types of input, X is converted into a UTF-8 string
-** and the string is hashed without the trailing 0x00 terminator.  The hash
-** of a NULL value is NULL.
-*/
-static void sha3Func(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  SHA3Context cx;
-  int eType = sqlite3_value_type(argv[0]);
-  int nByte = sqlite3_value_bytes(argv[0]);
-  int iSize;
-  if( argc==1 ){
-    iSize = 256;
-  }else{
-    iSize = sqlite3_value_int(argv[1]);
-    if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){
-      sqlite3_result_error(context, "SHA3 size should be one of: 224 256 "
-                                    "384 512", -1);
-      return;
-    }
-  }
-  if( eType==SQLITE_NULL ) return;
-  SHA3Init(&cx, iSize);
-  if( eType==SQLITE_BLOB ){
-    SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte);
-  }else{
-    SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte);
-  }
-  sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT);
-}
-
-/* Compute a string using sqlite3_vsnprintf() with a maximum length
-** of 50 bytes and add it to the hash.
-*/
-static void hash_step_vformat(
-  SHA3Context *p,                 /* Add content to this context */
-  const char *zFormat,
-  ...
-){
-  va_list ap;
-  int n;
-  char zBuf[50];
-  va_start(ap, zFormat);
-  sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap);
-  va_end(ap);
-  n = (int)strlen(zBuf);
-  SHA3Update(p, (unsigned char*)zBuf, n);
-}
-
-/*
-** Implementation of the sha3_query(SQL,SIZE) function.
-**
-** This function compiles and runs the SQL statement(s) given in the
-** argument. The results are hashed using a SIZE-bit SHA3.  The default
-** size is 256.
-**
-** The format of the byte stream that is hashed is summarized as follows:
-**
-**       S<n>:<sql>
-**       R
-**       N
-**       I<int>
-**       F<ieee-float>
-**       B<size>:<bytes>
-**       T<size>:<text>
-**
-** <sql> is the original SQL text for each statement run and <n> is
-** the size of that text.  The SQL text is UTF-8.  A single R character
-** occurs before the start of each row.  N means a NULL value.
-** I mean an 8-byte little-endian integer <int>.  F is a floating point
-** number with an 8-byte little-endian IEEE floating point value <ieee-float>.
-** B means blobs of <size> bytes.  T means text rendered as <size>
-** bytes of UTF-8.  The <n> and <size> values are expressed as an ASCII
-** text integers.
-**
-** For each SQL statement in the X input, there is one S segment.  Each
-** S segment is followed by zero or more R segments, one for each row in the
-** result set.  After each R, there are one or more N, I, F, B, or T segments,
-** one for each column in the result set.  Segments are concatentated directly
-** with no delimiters of any kind.
-*/
-static void sha3QueryFunc(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  sqlite3 *db = sqlite3_context_db_handle(context);
-  const char *zSql = (const char*)sqlite3_value_text(argv[0]);
-  sqlite3_stmt *pStmt = 0;
-  int nCol;                   /* Number of columns in the result set */
-  int i;                      /* Loop counter */
-  int rc;
-  int n;
-  const char *z;
-  SHA3Context cx;
-  int iSize;
-
-  if( argc==1 ){
-    iSize = 256;
-  }else{
-    iSize = sqlite3_value_int(argv[1]);
-    if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){
-      sqlite3_result_error(context, "SHA3 size should be one of: 224 256 "
-                                    "384 512", -1);
-      return;
-    }
-  }
-  if( zSql==0 ) return;
-  SHA3Init(&cx, iSize);
-  while( zSql[0] ){
-    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql);
-    if( rc ){
-      char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s",
-                                   zSql, sqlite3_errmsg(db));
-      sqlite3_finalize(pStmt);
-      sqlite3_result_error(context, zMsg, -1);
-      sqlite3_free(zMsg);
-      return;
-    }
-    if( !sqlite3_stmt_readonly(pStmt) ){
-      char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt));
-      sqlite3_finalize(pStmt);
-      sqlite3_result_error(context, zMsg, -1);
-      sqlite3_free(zMsg);
-      return;
-    }
-    nCol = sqlite3_column_count(pStmt);
-    z = sqlite3_sql(pStmt);
-    n = (int)strlen(z);
-    hash_step_vformat(&cx,"S%d:",n);
-    SHA3Update(&cx,(unsigned char*)z,n);
-
-    /* Compute a hash over the result of the query */
-    while( SQLITE_ROW==sqlite3_step(pStmt) ){
-      SHA3Update(&cx,(const unsigned char*)"R",1);
-      for(i=0; i<nCol; i++){
-        switch( sqlite3_column_type(pStmt,i) ){
-          case SQLITE_NULL: {
-            SHA3Update(&cx, (const unsigned char*)"N",1);
-            break;
-          }
-          case SQLITE_INTEGER: {
-            sqlite3_uint64 u;
-            int j;
-            unsigned char x[9];
-            sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
-            memcpy(&u, &v, 8);
-            for(j=8; j>=1; j--){
-              x[j] = u & 0xff;
-              u >>= 8;
-            }
-            x[0] = 'I';
-            SHA3Update(&cx, x, 9);
-            break;
-          }
-          case SQLITE_FLOAT: {
-            sqlite3_uint64 u;
-            int j;
-            unsigned char x[9];
-            double r = sqlite3_column_double(pStmt,i);
-            memcpy(&u, &r, 8);
-            for(j=8; j>=1; j--){
-              x[j] = u & 0xff;
-              u >>= 8;
-            }
-            x[0] = 'F';
-            SHA3Update(&cx,x,9);
-            break;
-          }
-          case SQLITE_TEXT: {
-            int n2 = sqlite3_column_bytes(pStmt, i);
-            const unsigned char *z2 = sqlite3_column_text(pStmt, i);
-            hash_step_vformat(&cx,"T%d:",n2);
-            SHA3Update(&cx, z2, n2);
-            break;
-          }
-          case SQLITE_BLOB: {
-            int n2 = sqlite3_column_bytes(pStmt, i);
-            const unsigned char *z2 = sqlite3_column_blob(pStmt, i);
-            hash_step_vformat(&cx,"B%d:",n2);
-            SHA3Update(&cx, z2, n2);
-            break;
-          }
-        }
-      }
-    }
-    sqlite3_finalize(pStmt);
-  }
-  sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT);
-}
-
-
-#ifdef _WIN32
-
-#endif
-int sqlite3_shathree_init(
-  sqlite3 *db,
-  char **pzErrMsg,
-  const sqlite3_api_routines *pApi
-){
-  int rc = SQLITE_OK;
-  SQLITE_EXTENSION_INIT2(pApi);
-  (void)pzErrMsg;  /* Unused parameter */
-  rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0,
-                               sha3Func, 0, 0);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0,
-                                 sha3Func, 0, 0);
-  }
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0,
-                                 sha3QueryFunc, 0, 0);
-  }
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0,
-                                 sha3QueryFunc, 0, 0);
-  }
-  return rc;
-}
-
-/************************* End ../ext/misc/shathree.c ********************/
-/************************* Begin ../ext/misc/fileio.c ******************/
-/*
-** 2014-06-13
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This SQLite extension implements SQL functions readfile() and
-** writefile().
-*/
-SQLITE_EXTENSION_INIT1
-#include <stdio.h>
-
-/*
-** Implementation of the "readfile(X)" SQL function.  The entire content
-** of the file named X is read and returned as a BLOB.  NULL is returned
-** if the file does not exist or is unreadable.
-*/
-static void readfileFunc(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  const char *zName;
-  FILE *in;
-  long nIn;
-  void *pBuf;
-
-  (void)(argc);  /* Unused parameter */
-  zName = (const char*)sqlite3_value_text(argv[0]);
-  if( zName==0 ) return;
-  in = fopen(zName, "rb");
-  if( in==0 ) return;
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc( nIn );
-  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
-    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
-  }else{
-    sqlite3_free(pBuf);
-  }
-  fclose(in);
-}
-
-/*
-** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
-** is written into file X.  The number of bytes written is returned.  Or
-** NULL is returned if something goes wrong, such as being unable to open
-** file X for writing.
-*/
-static void writefileFunc(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  FILE *out;
-  const char *z;
-  sqlite3_int64 rc;
-  const char *zFile;
-
-  (void)(argc);  /* Unused parameter */
-  zFile = (const char*)sqlite3_value_text(argv[0]);
-  if( zFile==0 ) return;
-  out = fopen(zFile, "wb");
-  if( out==0 ) return;
-  z = (const char*)sqlite3_value_blob(argv[1]);
-  if( z==0 ){
-    rc = 0;
-  }else{
-    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
-  }
-  fclose(out);
-  sqlite3_result_int64(context, rc);
-}
-
-
-#ifdef _WIN32
-
-#endif
-int sqlite3_fileio_init(
-  sqlite3 *db, 
-  char **pzErrMsg, 
-  const sqlite3_api_routines *pApi
-){
-  int rc = SQLITE_OK;
-  SQLITE_EXTENSION_INIT2(pApi);
-  (void)pzErrMsg;  /* Unused parameter */
-  rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
-                               readfileFunc, 0, 0);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
-                                 writefileFunc, 0, 0);
-  }
-  return rc;
-}
-
-/************************* End ../ext/misc/fileio.c ********************/
-/************************* Begin ../ext/misc/completion.c ******************/
-/*
-** 2017-07-10
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This file implements an eponymous virtual table that returns suggested
-** completions for a partial SQL input.
-**
-** Suggested usage:
-**
-**     SELECT DISTINCT candidate COLLATE nocase
-**       FROM completion($prefix,$wholeline)
-**      ORDER BY 1;
-**
-** The two query parameters are optional.  $prefix is the text of the
-** current word being typed and that is to be completed.  $wholeline is
-** the complete input line, used for context.
-**
-** The raw completion() table might return the same candidate multiple
-** times, for example if the same column name is used to two or more
-** tables.  And the candidates are returned in an arbitrary order.  Hence,
-** the DISTINCT and ORDER BY are recommended.
-**
-** This virtual table operates at the speed of human typing, and so there
-** is no attempt to make it fast.  Even a slow implementation will be much
-** faster than any human can type.
-**
-*/
-SQLITE_EXTENSION_INIT1
-#include <assert.h>
-#include <string.h>
-#include <ctype.h>
-
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-
-/* completion_vtab is a subclass of sqlite3_vtab which will
-** serve as the underlying representation of a completion virtual table
-*/
-typedef struct completion_vtab completion_vtab;
-struct completion_vtab {
-  sqlite3_vtab base;  /* Base class - must be first */
-  sqlite3 *db;        /* Database connection for this completion vtab */
-};
-
-/* completion_cursor is a subclass of sqlite3_vtab_cursor which will
-** serve as the underlying representation of a cursor that scans
-** over rows of the result
-*/
-typedef struct completion_cursor completion_cursor;
-struct completion_cursor {
-  sqlite3_vtab_cursor base;  /* Base class - must be first */
-  sqlite3 *db;               /* Database connection for this cursor */
-  int nPrefix, nLine;        /* Number of bytes in zPrefix and zLine */
-  char *zPrefix;             /* The prefix for the word we want to complete */
-  char *zLine;               /* The whole that we want to complete */
-  const char *zCurrentRow;   /* Current output row */
-  sqlite3_stmt *pStmt;       /* Current statement */
-  sqlite3_int64 iRowid;      /* The rowid */
-  int ePhase;                /* Current phase */
-  int j;                     /* inter-phase counter */
-};
-
-/* Values for ePhase:
-*/
-#define COMPLETION_FIRST_PHASE   1
-#define COMPLETION_KEYWORDS      1
-#define COMPLETION_PRAGMAS       2
-#define COMPLETION_FUNCTIONS     3
-#define COMPLETION_COLLATIONS    4
-#define COMPLETION_INDEXES       5
-#define COMPLETION_TRIGGERS      6
-#define COMPLETION_DATABASES     7
-#define COMPLETION_TABLES        8
-#define COMPLETION_COLUMNS       9
-#define COMPLETION_MODULES       10
-#define COMPLETION_EOF           11
-
-/*
-** The completionConnect() method is invoked to create a new
-** completion_vtab that describes the completion virtual table.
-**
-** Think of this routine as the constructor for completion_vtab objects.
-**
-** All this routine needs to do is:
-**
-**    (1) Allocate the completion_vtab object and initialize all fields.
-**
-**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
-**        result set of queries against completion will look like.
-*/
-static int completionConnect(
-  sqlite3 *db,
-  void *pAux,
-  int argc, const char *const*argv,
-  sqlite3_vtab **ppVtab,
-  char **pzErr
-){
-  completion_vtab *pNew;
-  int rc;
-
-  (void)(pAux);    /* Unused parameter */
-  (void)(argc);    /* Unused parameter */
-  (void)(argv);    /* Unused parameter */
-  (void)(pzErr);   /* Unused parameter */
-
-/* Column numbers */
-#define COMPLETION_COLUMN_CANDIDATE 0  /* Suggested completion of the input */
-#define COMPLETION_COLUMN_PREFIX    1  /* Prefix of the word to be completed */
-#define COMPLETION_COLUMN_WHOLELINE 2  /* Entire line seen so far */
-#define COMPLETION_COLUMN_PHASE     3  /* ePhase - used for debugging only */
-
-  rc = sqlite3_declare_vtab(db,
-      "CREATE TABLE x("
-      "  candidate TEXT,"
-      "  prefix TEXT HIDDEN,"
-      "  wholeline TEXT HIDDEN,"
-      "  phase INT HIDDEN"        /* Used for debugging only */
-      ")");
-  if( rc==SQLITE_OK ){
-    pNew = sqlite3_malloc( sizeof(*pNew) );
-    *ppVtab = (sqlite3_vtab*)pNew;
-    if( pNew==0 ) return SQLITE_NOMEM;
-    memset(pNew, 0, sizeof(*pNew));
-    pNew->db = db;
-  }
-  return rc;
-}
-
-/*
-** This method is the destructor for completion_cursor objects.
-*/
-static int completionDisconnect(sqlite3_vtab *pVtab){
-  sqlite3_free(pVtab);
-  return SQLITE_OK;
-}
-
-/*
-** Constructor for a new completion_cursor object.
-*/
-static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
-  completion_cursor *pCur;
-  pCur = sqlite3_malloc( sizeof(*pCur) );
-  if( pCur==0 ) return SQLITE_NOMEM;
-  memset(pCur, 0, sizeof(*pCur));
-  pCur->db = ((completion_vtab*)p)->db;
-  *ppCursor = &pCur->base;
-  return SQLITE_OK;
-}
-
-/*
-** Reset the completion_cursor.
-*/
-static void completionCursorReset(completion_cursor *pCur){
-  sqlite3_free(pCur->zPrefix);   pCur->zPrefix = 0;  pCur->nPrefix = 0;
-  sqlite3_free(pCur->zLine);     pCur->zLine = 0;    pCur->nLine = 0;
-  sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0;
-  pCur->j = 0;
-}
-
-/*
-** Destructor for a completion_cursor.
-*/
-static int completionClose(sqlite3_vtab_cursor *cur){
-  completionCursorReset((completion_cursor*)cur);
-  sqlite3_free(cur);
-  return SQLITE_OK;
-}
-
-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
-  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-  "WITH", "WITHOUT",
-};
-#define completionKwCount \
-   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
-/*
-** Advance a completion_cursor to its next row of output.
-**
-** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object
-** record the current state of the scan.  This routine sets ->zCurrentRow
-** to the current row of output and then returns.  If no more rows remain,
-** then ->ePhase is set to COMPLETION_EOF which will signal the virtual
-** table that has reached the end of its scan.
-**
-** The current implementation just lists potential identifiers and
-** keywords and filters them by zPrefix.  Future enhancements should
-** take zLine into account to try to restrict the set of identifiers and
-** keywords based on what would be legal at the current point of input.
-*/
-static int completionNext(sqlite3_vtab_cursor *cur){
-  completion_cursor *pCur = (completion_cursor*)cur;
-  int eNextPhase = 0;  /* Next phase to try if current phase reaches end */
-  int iCol = -1;       /* If >=0, step pCur->pStmt and use the i-th column */
-  pCur->iRowid++;
-  while( pCur->ePhase!=COMPLETION_EOF ){
-    switch( pCur->ePhase ){
-      case COMPLETION_KEYWORDS: {
-        if( pCur->j >= completionKwCount ){
-          pCur->zCurrentRow = 0;
-          pCur->ePhase = COMPLETION_DATABASES;
-        }else{
-          pCur->zCurrentRow = completionKwrds[pCur->j++];
-        }
-        iCol = -1;
-        break;
-      }
-      case COMPLETION_DATABASES: {
-        if( pCur->pStmt==0 ){
-          sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1,
-                             &pCur->pStmt, 0);
-        }
-        iCol = 1;
-        eNextPhase = COMPLETION_TABLES;
-        break;
-      }
-      case COMPLETION_TABLES: {
-        if( pCur->pStmt==0 ){
-          sqlite3_stmt *pS2;
-          char *zSql = 0;
-          const char *zSep = "";
-          sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0);
-          while( sqlite3_step(pS2)==SQLITE_ROW ){
-            const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
-            zSql = sqlite3_mprintf(
-               "%z%s"
-               "SELECT name FROM \"%w\".sqlite_master"
-               " WHERE type='table'",
-               zSql, zSep, zDb
-            );
-            if( zSql==0 ) return SQLITE_NOMEM;
-            zSep = " UNION ";
-          }
-          sqlite3_finalize(pS2);
-          sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0);
-          sqlite3_free(zSql);
-        }
-        iCol = 0;
-        eNextPhase = COMPLETION_COLUMNS;
-        break;
-      }
-      case COMPLETION_COLUMNS: {
-        if( pCur->pStmt==0 ){
-          sqlite3_stmt *pS2;
-          char *zSql = 0;
-          const char *zSep = "";
-          sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0);
-          while( sqlite3_step(pS2)==SQLITE_ROW ){
-            const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
-            zSql = sqlite3_mprintf(
-               "%z%s"
-               "SELECT pti.name FROM \"%w\".sqlite_master AS sm"
-                       " JOIN pragma_table_info(sm.name,%Q) AS pti"
-               " WHERE sm.type='table'",
-               zSql, zSep, zDb, zDb
-            );
-            if( zSql==0 ) return SQLITE_NOMEM;
-            zSep = " UNION ";
-          }
-          sqlite3_finalize(pS2);
-          sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0);
-          sqlite3_free(zSql);
-        }
-        iCol = 0;
-        eNextPhase = COMPLETION_EOF;
-        break;
-      }
-    }
-    if( iCol<0 ){
-      /* This case is when the phase presets zCurrentRow */
-      if( pCur->zCurrentRow==0 ) continue;
-    }else{
-      if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
-        /* Extract the next row of content */
-        pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
-      }else{
-        /* When all rows are finished, advance to the next phase */
-        sqlite3_finalize(pCur->pStmt);
-        pCur->pStmt = 0;
-        pCur->ePhase = eNextPhase;
-        continue;
-      }
-    }
-    if( pCur->nPrefix==0 ) break;
-    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
-      break;
-    }
-  }
-
-  return SQLITE_OK;
-}
-
-/*
-** Return values of columns for the row at which the completion_cursor
-** is currently pointing.
-*/
-static int completionColumn(
-  sqlite3_vtab_cursor *cur,   /* The cursor */
-  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
-  int i                       /* Which column to return */
-){
-  completion_cursor *pCur = (completion_cursor*)cur;
-  switch( i ){
-    case COMPLETION_COLUMN_CANDIDATE: {
-      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
-      break;
-    }
-    case COMPLETION_COLUMN_PREFIX: {
-      sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT);
-      break;
-    }
-    case COMPLETION_COLUMN_WHOLELINE: {
-      sqlite3_result_text(ctx, pCur->zLine, -1, SQLITE_TRANSIENT);
-      break;
-    }
-    case COMPLETION_COLUMN_PHASE: {
-      sqlite3_result_int(ctx, pCur->ePhase);
-      break;
-    }
-  }
-  return SQLITE_OK;
-}
-
-/*
-** Return the rowid for the current row.  In this implementation, the
-** rowid is the same as the output value.
-*/
-static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
-  completion_cursor *pCur = (completion_cursor*)cur;
-  *pRowid = pCur->iRowid;
-  return SQLITE_OK;
-}
-
-/*
-** Return TRUE if the cursor has been moved off of the last
-** row of output.
-*/
-static int completionEof(sqlite3_vtab_cursor *cur){
-  completion_cursor *pCur = (completion_cursor*)cur;
-  return pCur->ePhase >= COMPLETION_EOF;
-}
-
-/*
-** This method is called to "rewind" the completion_cursor object back
-** to the first row of output.  This method is always called at least
-** once prior to any call to completionColumn() or completionRowid() or 
-** completionEof().
-*/
-static int completionFilter(
-  sqlite3_vtab_cursor *pVtabCursor, 
-  int idxNum, const char *idxStr,
-  int argc, sqlite3_value **argv
-){
-  completion_cursor *pCur = (completion_cursor *)pVtabCursor;
-  int iArg = 0;
-  (void)(idxStr);   /* Unused parameter */
-  (void)(argc);     /* Unused parameter */
-  completionCursorReset(pCur);
-  if( idxNum & 1 ){
-    pCur->nPrefix = sqlite3_value_bytes(argv[iArg]);
-    if( pCur->nPrefix>0 ){
-      pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
-      if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
-    }
-    iArg++;
-  }
-  if( idxNum & 2 ){
-    pCur->nLine = sqlite3_value_bytes(argv[iArg]);
-    if( pCur->nLine>0 ){
-      pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
-      if( pCur->zLine==0 ) return SQLITE_NOMEM;
-    }
-    iArg++;
-  }
-  if( pCur->zLine!=0 && pCur->zPrefix==0 ){
-    int i = pCur->nLine;
-    while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){
-      i--;
-    }
-    pCur->nPrefix = pCur->nLine - i;
-    if( pCur->nPrefix>0 ){
-      pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i);
-      if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
-    }
-  }
-  pCur->iRowid = 0;
-  pCur->ePhase = COMPLETION_FIRST_PHASE;
-  return completionNext(pVtabCursor);
-}
-
-/*
-** SQLite will invoke this method one or more times while planning a query
-** that uses the completion virtual table.  This routine needs to create
-** a query plan for each invocation and compute an estimated cost for that
-** plan.
-**
-** There are two hidden parameters that act as arguments to the table-valued
-** function:  "prefix" and "wholeline".  Bit 0 of idxNum is set if "prefix"
-** is available and bit 1 is set if "wholeline" is available.
-*/
-static int completionBestIndex(
-  sqlite3_vtab *tab,
-  sqlite3_index_info *pIdxInfo
-){
-  int i;                 /* Loop over constraints */
-  int idxNum = 0;        /* The query plan bitmask */
-  int prefixIdx = -1;    /* Index of the start= constraint, or -1 if none */
-  int wholelineIdx = -1; /* Index of the stop= constraint, or -1 if none */
-  int nArg = 0;          /* Number of arguments that completeFilter() expects */
-  const struct sqlite3_index_constraint *pConstraint;
-
-  (void)(tab);    /* Unused parameter */
-  pConstraint = pIdxInfo->aConstraint;
-  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
-    if( pConstraint->usable==0 ) continue;
-    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    switch( pConstraint->iColumn ){
-      case COMPLETION_COLUMN_PREFIX:
-        prefixIdx = i;
-        idxNum |= 1;
-        break;
-      case COMPLETION_COLUMN_WHOLELINE:
-        wholelineIdx = i;
-        idxNum |= 2;
-        break;
-    }
-  }
-  if( prefixIdx>=0 ){
-    pIdxInfo->aConstraintUsage[prefixIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[prefixIdx].omit = 1;
-  }
-  if( wholelineIdx>=0 ){
-    pIdxInfo->aConstraintUsage[wholelineIdx].argvIndex = ++nArg;
-    pIdxInfo->aConstraintUsage[wholelineIdx].omit = 1;
-  }
-  pIdxInfo->idxNum = idxNum;
-  pIdxInfo->estimatedCost = (double)5000 - 1000*nArg;
-  pIdxInfo->estimatedRows = 500 - 100*nArg;
-  return SQLITE_OK;
-}
-
-/*
-** This following structure defines all the methods for the 
-** completion virtual table.
-*/
-static sqlite3_module completionModule = {
-  0,                         /* iVersion */
-  0,                         /* xCreate */
-  completionConnect,         /* xConnect */
-  completionBestIndex,       /* xBestIndex */
-  completionDisconnect,      /* xDisconnect */
-  0,                         /* xDestroy */
-  completionOpen,            /* xOpen - open a cursor */
-  completionClose,           /* xClose - close a cursor */
-  completionFilter,          /* xFilter - configure scan constraints */
-  completionNext,            /* xNext - advance a cursor */
-  completionEof,             /* xEof - check for end of scan */
-  completionColumn,          /* xColumn - read data */
-  completionRowid,           /* xRowid - read data */
-  0,                         /* xUpdate */
-  0,                         /* xBegin */
-  0,                         /* xSync */
-  0,                         /* xCommit */
-  0,                         /* xRollback */
-  0,                         /* xFindMethod */
-  0,                         /* xRename */
-  0,                         /* xSavepoint */
-  0,                         /* xRelease */
-  0                          /* xRollbackTo */
-};
-
-#endif /* SQLITE_OMIT_VIRTUALTABLE */
-
-int sqlite3CompletionVtabInit(sqlite3 *db){
-  int rc = SQLITE_OK;
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  rc = sqlite3_create_module(db, "completion", &completionModule, 0);
-#endif
-  return rc;
-}
-
-#ifdef _WIN32
-
-#endif
-int sqlite3_completion_init(
-  sqlite3 *db, 
-  char **pzErrMsg, 
-  const sqlite3_api_routines *pApi
-){
-  int rc = SQLITE_OK;
-  SQLITE_EXTENSION_INIT2(pApi);
-  (void)(pzErrMsg);  /* Unused parameter */
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  rc = sqlite3CompletionVtabInit(db);
-#endif
-  return rc;
-}
-
-/************************* End ../ext/misc/completion.c ********************/
-
-#if defined(SQLITE_ENABLE_SESSION)
-/*
-** State information for a single open session
-*/
-typedef struct OpenSession OpenSession;
-struct OpenSession {
-  char *zName;             /* Symbolic name for this session */
-  int nFilter;             /* Number of xFilter rejection GLOB patterns */
-  char **azFilter;         /* Array of xFilter rejection GLOB patterns */
-  sqlite3_session *p;      /* The open session */
-};
-#endif
-
-/*
-** Shell output mode information from before ".explain on",
-** saved so that it can be restored by ".explain off"
-*/
-typedef struct SavedModeInfo SavedModeInfo;
-struct SavedModeInfo {
-  int valid;          /* Is there legit data in here? */
-  int mode;           /* Mode prior to ".explain on" */
-  int showHeader;     /* The ".header" setting prior to ".explain on" */
-  int colWidth[100];  /* Column widths prior to ".explain on" */
-};
-
-/*
-** State information about the database connection is contained in an
-** instance of the following structure.
-*/
-typedef struct ShellState ShellState;
-struct ShellState {
-  sqlite3 *db;           /* The database */
-  int autoExplain;       /* Automatically turn on .explain mode */
-  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
-  int statsOn;           /* True to display memory stats before each finalize */
-  int scanstatsOn;       /* True to display scan stats before each finalize */
-  int outCount;          /* Revert to stdout when reaching zero */
-  int cnt;               /* Number of records displayed so far */
-  FILE *out;             /* Write results here */
-  FILE *traceOut;        /* Output for sqlite3_trace() */
-  int nErr;              /* Number of errors seen */
-  int mode;              /* An output mode setting */
-  int cMode;             /* temporary output mode for the current query */
-  int normalMode;        /* Output mode before ".explain on" */
-  int writableSchema;    /* True if PRAGMA writable_schema=ON */
-  int showHeader;        /* True to show column names in List or Column mode */
-  int nCheck;            /* Number of ".check" commands run */
-  unsigned shellFlgs;    /* Various flags */
-  char *zDestTable;      /* Name of destination table when MODE_Insert */
-  char zTestcase[30];    /* Name of current test case */
-  char colSeparator[20]; /* Column separator character for several modes */
-  char rowSeparator[20]; /* Row separator character for MODE_Ascii */
-  int colWidth[100];     /* Requested width of each column when in column mode*/
-  int actualWidth[100];  /* Actual width of each column */
-  char nullValue[20];    /* The text to print when a NULL comes back from
-                         ** the database */
-  char outfile[FILENAME_MAX]; /* Filename for *out */
-  const char *zDbFilename;    /* name of the database file */
-  char *zFreeOnClose;         /* Filename to free when closing */
-  const char *zVfs;           /* Name of VFS to use */
-  sqlite3_stmt *pStmt;   /* Current statement if any. */
-  FILE *pLog;            /* Write log output here */
-  int *aiIndent;         /* Array of indents used in MODE_Explain */
-  int nIndent;           /* Size of array aiIndent[] */
-  int iIndent;           /* Index of current op in aiIndent[] */
-#if defined(SQLITE_ENABLE_SESSION)
-  int nSession;             /* Number of active sessions */
-  OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
-#endif
-};
-
-/*
-** These are the allowed shellFlgs values
-*/
-#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
-#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
-#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
-#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
-#define SHFLG_Newlines       0x00000010 /* .dump --newline flag */
-#define SHFLG_CountChanges   0x00000020 /* .changes setting */
-#define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
-
-/*
-** Macros for testing and setting shellFlgs
-*/
-#define ShellHasFlag(P,X)    (((P)->shellFlgs & (X))!=0)
-#define ShellSetFlag(P,X)    ((P)->shellFlgs|=(X))
-#define ShellClearFlag(P,X)  ((P)->shellFlgs&=(~(X)))
-
-/*
-** These are the allowed modes.
-*/
-#define MODE_Line     0  /* One column per line.  Blank line between records */
-#define MODE_Column   1  /* One record per line in neat columns */
-#define MODE_List     2  /* One record per line with a separator */
-#define MODE_Semi     3  /* Same as MODE_List but append ";" to each line */
-#define MODE_Html     4  /* Generate an XHTML table */
-#define MODE_Insert   5  /* Generate SQL "insert" statements */
-#define MODE_Quote    6  /* Quote values as for SQL */
-#define MODE_Tcl      7  /* Generate ANSI-C or TCL quoted elements */
-#define MODE_Csv      8  /* Quote strings, numbers are plain */
-#define MODE_Explain  9  /* Like MODE_Column, but do not truncate data */
-#define MODE_Ascii   10  /* Use ASCII unit and record separators (0x1F/0x1E) */
-#define MODE_Pretty  11  /* Pretty-print schemas */
-
-static const char *modeDescr[] = {
-  "line",
-  "column",
-  "list",
-  "semi",
-  "html",
-  "insert",
-  "quote",
-  "tcl",
-  "csv",
-  "explain",
-  "ascii",
-  "prettyprint",
-};
-
-/*
-** These are the column/row/line separators used by the various
-** import/export modes.
-*/
-#define SEP_Column    "|"
-#define SEP_Row       "\n"
-#define SEP_Tab       "\t"
-#define SEP_Space     " "
-#define SEP_Comma     ","
-#define SEP_CrLf      "\r\n"
-#define SEP_Unit      "\x1F"
-#define SEP_Record    "\x1E"
-
-/*
-** Number of elements in an array
-*/
-#define ArraySize(X)  (int)(sizeof(X)/sizeof(X[0]))
-
-/*
-** A callback for the sqlite3_log() interface.
-*/
-static void shellLog(void *pArg, int iErrCode, const char *zMsg){
-  ShellState *p = (ShellState*)pArg;
-  if( p->pLog==0 ) return;
-  utf8_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg);
-  fflush(p->pLog);
-}
-
-/*
-** Output the given string as a hex-encoded blob (eg. X'1234' )
-*/
-static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){
-  int i;
-  char *zBlob = (char *)pBlob;
-  raw_printf(out,"X'");
-  for(i=0; i<nBlob; i++){ raw_printf(out,"%02x",zBlob[i]&0xff); }
-  raw_printf(out,"'");
-}
-
-/*
-** Find a string that is not found anywhere in z[].  Return a pointer
-** to that string.
-**
-** Try to use zA and zB first.  If both of those are already found in z[]
-** then make up some string and store it in the buffer zBuf.
-*/
-static const char *unused_string(
-  const char *z,                    /* Result must not appear anywhere in z */
-  const char *zA, const char *zB,   /* Try these first */
-  char *zBuf                        /* Space to store a generated string */
-){
-  unsigned i = 0;
-  if( strstr(z, zA)==0 ) return zA;
-  if( strstr(z, zB)==0 ) return zB;
-  do{
-    sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
-  }while( strstr(z,zBuf)!=0 );
-  return zBuf;
-}
-
-/*
-** Output the given string as a quoted string using SQL quoting conventions.
-**
-** See also: output_quoted_escaped_string()
-*/
-static void output_quoted_string(FILE *out, const char *z){
-  int i;
-  char c;
-  setBinaryMode(out, 1);
-  for(i=0; (c = z[i])!=0 && c!='\''; i++){}
-  if( c==0 ){
-    utf8_printf(out,"'%s'",z);
-  }else{
-    raw_printf(out, "'");
-    while( *z ){
-      for(i=0; (c = z[i])!=0 && c!='\''; i++){}
-      if( c=='\'' ) i++;
-      if( i ){
-        utf8_printf(out, "%.*s", i, z);
-        z += i;
-      }
-      if( c=='\'' ){
-        raw_printf(out, "'");
-        continue;
-      }
-      if( c==0 ){
-        break;
-      }
-      z++;
-    }
-    raw_printf(out, "'");
-  }
-  setTextMode(out, 1);
-}
-
-/*
-** Output the given string as a quoted string using SQL quoting conventions.
-** Additionallly , escape the "\n" and "\r" characters so that they do not
-** get corrupted by end-of-line translation facilities in some operating
-** systems.
-**
-** This is like output_quoted_string() but with the addition of the \r\n
-** escape mechanism.
-*/
-static void output_quoted_escaped_string(FILE *out, const char *z){
-  int i;
-  char c;
-  setBinaryMode(out, 1);
-  for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){}
-  if( c==0 ){
-    utf8_printf(out,"'%s'",z);
-  }else{
-    const char *zNL = 0;
-    const char *zCR = 0;
-    int nNL = 0;
-    int nCR = 0;
-    char zBuf1[20], zBuf2[20];
-    for(i=0; z[i]; i++){
-      if( z[i]=='\n' ) nNL++;
-      if( z[i]=='\r' ) nCR++;
-    }
-    if( nNL ){
-      raw_printf(out, "replace(");
-      zNL = unused_string(z, "\\n", "\\012", zBuf1);
-    }
-    if( nCR ){
-      raw_printf(out, "replace(");
-      zCR = unused_string(z, "\\r", "\\015", zBuf2);
-    }
-    raw_printf(out, "'");
-    while( *z ){
-      for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
-      if( c=='\'' ) i++;
-      if( i ){
-        utf8_printf(out, "%.*s", i, z);
-        z += i;
-      }
-      if( c=='\'' ){
-        raw_printf(out, "'");
-        continue;
-      }
-      if( c==0 ){
-        break;
-      }
-      z++;
-      if( c=='\n' ){
-        raw_printf(out, "%s", zNL);
-        continue;
-      }
-      raw_printf(out, "%s", zCR);
-    }
-    raw_printf(out, "'");
-    if( nCR ){
-      raw_printf(out, ",'%s',char(13))", zCR);
-    }
-    if( nNL ){
-      raw_printf(out, ",'%s',char(10))", zNL);
-    }
-  }
-  setTextMode(out, 1);
-}
-
-/*
-** Output the given string as a quoted according to C or TCL quoting rules.
-*/
-static void output_c_string(FILE *out, const char *z){
-  unsigned int c;
-  fputc('"', out);
-  while( (c = *(z++))!=0 ){
-    if( c=='\\' ){
-      fputc(c, out);
-      fputc(c, out);
-    }else if( c=='"' ){
-      fputc('\\', out);
-      fputc('"', out);
-    }else if( c=='\t' ){
-      fputc('\\', out);
-      fputc('t', out);
-    }else if( c=='\n' ){
-      fputc('\\', out);
-      fputc('n', out);
-    }else if( c=='\r' ){
-      fputc('\\', out);
-      fputc('r', out);
-    }else if( !isprint(c&0xff) ){
-      raw_printf(out, "\\%03o", c&0xff);
-    }else{
-      fputc(c, out);
-    }
-  }
-  fputc('"', out);
-}
-
-/*
-** Output the given string with characters that are special to
-** HTML escaped.
-*/
-static void output_html_string(FILE *out, const char *z){
-  int i;
-  if( z==0 ) z = "";
-  while( *z ){
-    for(i=0;   z[i]
-            && z[i]!='<'
-            && z[i]!='&'
-            && z[i]!='>'
-            && z[i]!='\"'
-            && z[i]!='\'';
-        i++){}
-    if( i>0 ){
-      utf8_printf(out,"%.*s",i,z);
-    }
-    if( z[i]=='<' ){
-      raw_printf(out,"&lt;");
-    }else if( z[i]=='&' ){
-      raw_printf(out,"&amp;");
-    }else if( z[i]=='>' ){
-      raw_printf(out,"&gt;");
-    }else if( z[i]=='\"' ){
-      raw_printf(out,"&quot;");
-    }else if( z[i]=='\'' ){
-      raw_printf(out,"&#39;");
-    }else{
-      break;
-    }
-    z += i + 1;
-  }
-}
-
-/*
-** If a field contains any character identified by a 1 in the following
-** array, then the string must be quoted for CSV.
-*/
-static const char needCsvQuote[] = {
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 0, 1, 0, 0, 0, 0, 1,   0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,
-};
-
-/*
-** Output a single term of CSV.  Actually, p->colSeparator is used for
-** the separator, which may or may not be a comma.  p->nullValue is
-** the null value.  Strings are quoted if necessary.  The separator
-** is only issued if bSep is true.
-*/
-static void output_csv(ShellState *p, const char *z, int bSep){
-  FILE *out = p->out;
-  if( z==0 ){
-    utf8_printf(out,"%s",p->nullValue);
-  }else{
-    int i;
-    int nSep = strlen30(p->colSeparator);
-    for(i=0; z[i]; i++){
-      if( needCsvQuote[((unsigned char*)z)[i]]
-         || (z[i]==p->colSeparator[0] &&
-             (nSep==1 || memcmp(z, p->colSeparator, nSep)==0)) ){
-        i = 0;
-        break;
-      }
-    }
-    if( i==0 ){
-      putc('"', out);
-      for(i=0; z[i]; i++){
-        if( z[i]=='"' ) putc('"', out);
-        putc(z[i], out);
-      }
-      putc('"', out);
-    }else{
-      utf8_printf(out, "%s", z);
-    }
-  }
-  if( bSep ){
-    utf8_printf(p->out, "%s", p->colSeparator);
-  }
-}
-
-#ifdef SIGINT
-/*
-** This routine runs when the user presses Ctrl-C
-*/
-static void interrupt_handler(int NotUsed){
-  UNUSED_PARAMETER(NotUsed);
-  seenInterrupt++;
-  if( seenInterrupt>2 ) exit(1);
-  if( globalDb ) sqlite3_interrupt(globalDb);
-}
-#endif
-
-#ifndef SQLITE_OMIT_AUTHORIZATION
-/*
-** When the ".auth ON" is set, the following authorizer callback is
-** invoked.  It always returns SQLITE_OK.
-*/
-static int shellAuth(
-  void *pClientData,
-  int op,
-  const char *zA1,
-  const char *zA2,
-  const char *zA3,
-  const char *zA4
-){
-  ShellState *p = (ShellState*)pClientData;
-  static const char *azAction[] = { 0,
-     "CREATE_INDEX",         "CREATE_TABLE",         "CREATE_TEMP_INDEX",
-     "CREATE_TEMP_TABLE",    "CREATE_TEMP_TRIGGER",  "CREATE_TEMP_VIEW",
-     "CREATE_TRIGGER",       "CREATE_VIEW",          "DELETE",
-     "DROP_INDEX",           "DROP_TABLE",           "DROP_TEMP_INDEX",
-     "DROP_TEMP_TABLE",      "DROP_TEMP_TRIGGER",    "DROP_TEMP_VIEW",
-     "DROP_TRIGGER",         "DROP_VIEW",            "INSERT",
-     "PRAGMA",               "READ",                 "SELECT",
-     "TRANSACTION",          "UPDATE",               "ATTACH",
-     "DETACH",               "ALTER_TABLE",          "REINDEX",
-     "ANALYZE",              "CREATE_VTABLE",        "DROP_VTABLE",
-     "FUNCTION",             "SAVEPOINT",            "RECURSIVE"
-  };
-  int i;
-  const char *az[4];
-  az[0] = zA1;
-  az[1] = zA2;
-  az[2] = zA3;
-  az[3] = zA4;
-  utf8_printf(p->out, "authorizer: %s", azAction[op]);
-  for(i=0; i<4; i++){
-    raw_printf(p->out, " ");
-    if( az[i] ){
-      output_c_string(p->out, az[i]);
-    }else{
-      raw_printf(p->out, "NULL");
-    }
-  }
-  raw_printf(p->out, "\n");
-  return SQLITE_OK;
-}
-#endif
-
-/*
-** Print a schema statement.  Part of MODE_Semi and MODE_Pretty output.
-**
-** This routine converts some CREATE TABLE statements for shadow tables
-** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements.
-*/
-static void printSchemaLine(FILE *out, const char *z, const char *zTail){
-  if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){
-    utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail);
-  }else{
-    utf8_printf(out, "%s%s", z, zTail);
-  }
-}
-static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){
-  char c = z[n];
-  z[n] = 0;
-  printSchemaLine(out, z, zTail);
-  z[n] = c;
-}
-
-/*
-** This is the callback routine that the shell
-** invokes for each row of a query result.
-*/
-static int shell_callback(
-  void *pArg,
-  int nArg,        /* Number of result columns */
-  char **azArg,    /* Text of each result column */
-  char **azCol,    /* Column names */
-  int *aiType      /* Column types */
-){
-  int i;
-  ShellState *p = (ShellState*)pArg;
-
-  if( azArg==0 ) return 0;
-  switch( p->cMode ){
-    case MODE_Line: {
-      int w = 5;
-      if( azArg==0 ) break;
-      for(i=0; i<nArg; i++){
-        int len = strlen30(azCol[i] ? azCol[i] : "");
-        if( len>w ) w = len;
-      }
-      if( p->cnt++>0 ) utf8_printf(p->out, "%s", p->rowSeparator);
-      for(i=0; i<nArg; i++){
-        utf8_printf(p->out,"%*s = %s%s", w, azCol[i],
-                azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator);
-      }
-      break;
-    }
-    case MODE_Explain:
-    case MODE_Column: {
-      static const int aExplainWidths[] = {4, 13, 4, 4, 4, 13, 2, 13};
-      const int *colWidth;
-      int showHdr;
-      char *rowSep;
-      if( p->cMode==MODE_Column ){
-        colWidth = p->colWidth;
-        showHdr = p->showHeader;
-        rowSep = p->rowSeparator;
-      }else{
-        colWidth = aExplainWidths;
-        showHdr = 1;
-        rowSep = SEP_Row;
-      }
-      if( p->cnt++==0 ){
-        for(i=0; i<nArg; i++){
-          int w, n;
-          if( i<ArraySize(p->colWidth) ){
-            w = colWidth[i];
-          }else{
-            w = 0;
-          }
-          if( w==0 ){
-            w = strlenChar(azCol[i] ? azCol[i] : "");
-            if( w<10 ) w = 10;
-            n = strlenChar(azArg && azArg[i] ? azArg[i] : p->nullValue);
-            if( w<n ) w = n;
-          }
-          if( i<ArraySize(p->actualWidth) ){
-            p->actualWidth[i] = w;
-          }
-          if( showHdr ){
-            utf8_width_print(p->out, w, azCol[i]);
-            utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
-          }
-        }
-        if( showHdr ){
-          for(i=0; i<nArg; i++){
-            int w;
-            if( i<ArraySize(p->actualWidth) ){
-               w = p->actualWidth[i];
-               if( w<0 ) w = -w;
-            }else{
-               w = 10;
-            }
-            utf8_printf(p->out,"%-*.*s%s",w,w,
-                   "----------------------------------------------------------"
-                   "----------------------------------------------------------",
-                    i==nArg-1 ? rowSep : "  ");
-          }
-        }
-      }
-      if( azArg==0 ) break;
-      for(i=0; i<nArg; i++){
-        int w;
-        if( i<ArraySize(p->actualWidth) ){
-           w = p->actualWidth[i];
-        }else{
-           w = 10;
-        }
-        if( p->cMode==MODE_Explain && azArg[i] && strlenChar(azArg[i])>w ){
-          w = strlenChar(azArg[i]);
-        }
-        if( i==1 && p->aiIndent && p->pStmt ){
-          if( p->iIndent<p->nIndent ){
-            utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
-          }
-          p->iIndent++;
-        }
-        utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue);
-        utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
-      }
-      break;
-    }
-    case MODE_Semi: {   /* .schema and .fullschema output */
-      printSchemaLine(p->out, azArg[0], ";\n");
-      break;
-    }
-    case MODE_Pretty: {  /* .schema and .fullschema with --indent */
-      char *z;
-      int j;
-      int nParen = 0;
-      char cEnd = 0;
-      char c;
-      int nLine = 0;
-      assert( nArg==1 );
-      if( azArg[0]==0 ) break;
-      if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0
-       || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0
-      ){
-        utf8_printf(p->out, "%s;\n", azArg[0]);
-        break;
-      }
-      z = sqlite3_mprintf("%s", azArg[0]);
-      j = 0;
-      for(i=0; IsSpace(z[i]); i++){}
-      for(; (c = z[i])!=0; i++){
-        if( IsSpace(c) ){
-          if( z[j-1]=='\r' ) z[j-1] = '\n';
-          if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue;
-        }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){
-          j--;
-        }
-        z[j++] = c;
-      }
-      while( j>0 && IsSpace(z[j-1]) ){ j--; }
-      z[j] = 0;
-      if( strlen30(z)>=79 ){
-        for(i=j=0; (c = z[i])!=0; i++){
-          if( c==cEnd ){
-            cEnd = 0;
-          }else if( c=='"' || c=='\'' || c=='`' ){
-            cEnd = c;
-          }else if( c=='[' ){
-            cEnd = ']';
-          }else if( c=='(' ){
-            nParen++;
-          }else if( c==')' ){
-            nParen--;
-            if( nLine>0 && nParen==0 && j>0 ){
-              printSchemaLineN(p->out, z, j, "\n");
-              j = 0;
-            }
-          }
-          z[j++] = c;
-          if( nParen==1 && (c=='(' || c==',' || c=='\n') ){
-            if( c=='\n' ) j--;
-            printSchemaLineN(p->out, z, j, "\n  ");
-            j = 0;
-            nLine++;
-            while( IsSpace(z[i+1]) ){ i++; }
-          }
-        }
-        z[j] = 0;
-      }
-      printSchemaLine(p->out, z, ";\n");
-      sqlite3_free(z);
-      break;
-    }
-    case MODE_List: {
-      if( p->cnt++==0 && p->showHeader ){
-        for(i=0; i<nArg; i++){
-          utf8_printf(p->out,"%s%s",azCol[i],
-                  i==nArg-1 ? p->rowSeparator : p->colSeparator);
-        }
-      }
-      if( azArg==0 ) break;
-      for(i=0; i<nArg; i++){
-        char *z = azArg[i];
-        if( z==0 ) z = p->nullValue;
-        utf8_printf(p->out, "%s", z);
-        if( i<nArg-1 ){
-          utf8_printf(p->out, "%s", p->colSeparator);
-        }else{
-          utf8_printf(p->out, "%s", p->rowSeparator);
-        }
-      }
-      break;
-    }
-    case MODE_Html: {
-      if( p->cnt++==0 && p->showHeader ){
-        raw_printf(p->out,"<TR>");
-        for(i=0; i<nArg; i++){
-          raw_printf(p->out,"<TH>");
-          output_html_string(p->out, azCol[i]);
-          raw_printf(p->out,"</TH>\n");
-        }
-        raw_printf(p->out,"</TR>\n");
-      }
-      if( azArg==0 ) break;
-      raw_printf(p->out,"<TR>");
-      for(i=0; i<nArg; i++){
-        raw_printf(p->out,"<TD>");
-        output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue);
-        raw_printf(p->out,"</TD>\n");
-      }
-      raw_printf(p->out,"</TR>\n");
-      break;
-    }
-    case MODE_Tcl: {
-      if( p->cnt++==0 && p->showHeader ){
-        for(i=0; i<nArg; i++){
-          output_c_string(p->out,azCol[i] ? azCol[i] : "");
-          if(i<nArg-1) utf8_printf(p->out, "%s", p->colSeparator);
-        }
-        utf8_printf(p->out, "%s", p->rowSeparator);
-      }
-      if( azArg==0 ) break;
-      for(i=0; i<nArg; i++){
-        output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue);
-        if(i<nArg-1) utf8_printf(p->out, "%s", p->colSeparator);
-      }
-      utf8_printf(p->out, "%s", p->rowSeparator);
-      break;
-    }
-    case MODE_Csv: {
-      setBinaryMode(p->out, 1);
-      if( p->cnt++==0 && p->showHeader ){
-        for(i=0; i<nArg; i++){
-          output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1);
-        }
-        utf8_printf(p->out, "%s", p->rowSeparator);
-      }
-      if( nArg>0 ){
-        for(i=0; i<nArg; i++){
-          output_csv(p, azArg[i], i<nArg-1);
-        }
-        utf8_printf(p->out, "%s", p->rowSeparator);
-      }
-      setTextMode(p->out, 1);
-      break;
-    }
-    case MODE_Insert: {
-      if( azArg==0 ) break;
-      utf8_printf(p->out,"INSERT INTO %s",p->zDestTable);
-      if( p->showHeader ){
-        raw_printf(p->out,"(");
-        for(i=0; i<nArg; i++){
-          if( i>0 ) raw_printf(p->out, ",");
-          if( quoteChar(azCol[i]) ){
-            char *z = sqlite3_mprintf("\"%w\"", azCol[i]);
-            utf8_printf(p->out, "%s", z);
-            sqlite3_free(z);
-          }else{
-            raw_printf(p->out, "%s", azCol[i]);
-          }
-        }
-        raw_printf(p->out,")");
-      }
-      p->cnt++;
-      for(i=0; i<nArg; i++){
-        raw_printf(p->out, i>0 ? "," : " VALUES(");
-        if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){
-          utf8_printf(p->out,"NULL");
-        }else if( aiType && aiType[i]==SQLITE_TEXT ){
-          if( ShellHasFlag(p, SHFLG_Newlines) ){
-            output_quoted_string(p->out, azArg[i]);
-          }else{
-            output_quoted_escaped_string(p->out, azArg[i]);
-          }
-        }else if( aiType && aiType[i]==SQLITE_INTEGER ){
-          utf8_printf(p->out,"%s", azArg[i]);
-        }else if( aiType && aiType[i]==SQLITE_FLOAT ){
-          char z[50];
-          double r = sqlite3_column_double(p->pStmt, i);
-          sqlite3_snprintf(50,z,"%!.20g", r);
-          raw_printf(p->out, "%s", z);
-        }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){
-          const void *pBlob = sqlite3_column_blob(p->pStmt, i);
-          int nBlob = sqlite3_column_bytes(p->pStmt, i);
-          output_hex_blob(p->out, pBlob, nBlob);
-        }else if( isNumber(azArg[i], 0) ){
-          utf8_printf(p->out,"%s", azArg[i]);
-        }else if( ShellHasFlag(p, SHFLG_Newlines) ){
-          output_quoted_string(p->out, azArg[i]);
-        }else{
-          output_quoted_escaped_string(p->out, azArg[i]);
-        }
-      }
-      raw_printf(p->out,");\n");
-      break;
-    }
-    case MODE_Quote: {
-      if( azArg==0 ) break;
-      if( p->cnt==0 && p->showHeader ){
-        for(i=0; i<nArg; i++){
-          if( i>0 ) raw_printf(p->out, ",");
-          output_quoted_string(p->out, azCol[i]);
-        }
-        raw_printf(p->out,"\n");
-      }
-      p->cnt++;
-      for(i=0; i<nArg; i++){
-        if( i>0 ) raw_printf(p->out, ",");
-        if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){
-          utf8_printf(p->out,"NULL");
-        }else if( aiType && aiType[i]==SQLITE_TEXT ){
-          output_quoted_string(p->out, azArg[i]);
-        }else if( aiType && aiType[i]==SQLITE_INTEGER ){
-          utf8_printf(p->out,"%s", azArg[i]);
-        }else if( aiType && aiType[i]==SQLITE_FLOAT ){
-          char z[50];
-          double r = sqlite3_column_double(p->pStmt, i);
-          sqlite3_snprintf(50,z,"%!.20g", r);
-          raw_printf(p->out, "%s", z);
-        }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){
-          const void *pBlob = sqlite3_column_blob(p->pStmt, i);
-          int nBlob = sqlite3_column_bytes(p->pStmt, i);
-          output_hex_blob(p->out, pBlob, nBlob);
-        }else if( isNumber(azArg[i], 0) ){
-          utf8_printf(p->out,"%s", azArg[i]);
-        }else{
-          output_quoted_string(p->out, azArg[i]);
-        }
-      }
-      raw_printf(p->out,"\n");
-      break;
-    }
-    case MODE_Ascii: {
-      if( p->cnt++==0 && p->showHeader ){
-        for(i=0; i<nArg; i++){
-          if( i>0 ) utf8_printf(p->out, "%s", p->colSeparator);
-          utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : "");
-        }
-        utf8_printf(p->out, "%s", p->rowSeparator);
-      }
-      if( azArg==0 ) break;
-      for(i=0; i<nArg; i++){
-        if( i>0 ) utf8_printf(p->out, "%s", p->colSeparator);
-        utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue);
-      }
-      utf8_printf(p->out, "%s", p->rowSeparator);
-      break;
-    }
-  }
-  return 0;
-}
-
-/*
-** This is the callback routine that the SQLite library
-** invokes for each row of a query result.
-*/
-static int callback(void *pArg, int nArg, char **azArg, char **azCol){
-  /* since we don't have type info, call the shell_callback with a NULL value */
-  return shell_callback(pArg, nArg, azArg, azCol, NULL);
-}
-
-/*
-** This is the callback routine from sqlite3_exec() that appends all
-** output onto the end of a ShellText object.
-*/
-static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){
-  ShellText *p = (ShellText*)pArg;
-  int i;
-  UNUSED_PARAMETER(az);
-  if( azArg==0 ) return 0;
-  if( p->n ) appendText(p, "|", 0);
-  for(i=0; i<nArg; i++){
-    if( i ) appendText(p, ",", 0);
-    if( azArg[i] ) appendText(p, azArg[i], 0);
-  }
-  return 0;
-}
-
-/*
-** Generate an appropriate SELFTEST table in the main database.
-*/
-static void createSelftestTable(ShellState *p){
-  char *zErrMsg = 0;
-  sqlite3_exec(p->db,
-    "SAVEPOINT selftest_init;\n"
-    "CREATE TABLE IF NOT EXISTS selftest(\n"
-    "  tno INTEGER PRIMARY KEY,\n"   /* Test number */
-    "  op TEXT,\n"                   /* Operator:  memo run */
-    "  cmd TEXT,\n"                  /* Command text */
-    "  ans TEXT\n"                   /* Desired answer */
-    ");"
-    "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n"
-    "INSERT INTO [_shell$self](rowid,op,cmd)\n"
-    "  VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n"
-    "         'memo','Tests generated by --init');\n"
-    "INSERT INTO [_shell$self]\n"
-    "  SELECT 'run',\n"
-    "    'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql "
-                                 "FROM sqlite_master ORDER BY 2'',224))',\n"
-    "    hex(sha3_query('SELECT type,name,tbl_name,sql "
-                          "FROM sqlite_master ORDER BY 2',224));\n"
-    "INSERT INTO [_shell$self]\n"
-    "  SELECT 'run',"
-    "    'SELECT hex(sha3_query(''SELECT * FROM \"' ||"
-    "        printf('%w',name) || '\" NOT INDEXED'',224))',\n"
-    "    hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n"
-    "  FROM (\n"
-    "    SELECT name FROM sqlite_master\n"
-    "     WHERE type='table'\n"
-    "       AND name<>'selftest'\n"
-    "       AND coalesce(rootpage,0)>0\n"
-    "  )\n"
-    " ORDER BY name;\n"
-    "INSERT INTO [_shell$self]\n"
-    "  VALUES('run','PRAGMA integrity_check','ok');\n"
-    "INSERT INTO selftest(tno,op,cmd,ans)"
-    "  SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n"
-    "DROP TABLE [_shell$self];"
-    ,0,0,&zErrMsg);
-  if( zErrMsg ){
-    utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg);
-    sqlite3_free(zErrMsg);
-  }
-  sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0);
-}
-
-
-/*
-** Set the destination table field of the ShellState structure to
-** the name of the table given.  Escape any quote characters in the
-** table name.
-*/
-static void set_table_name(ShellState *p, const char *zName){
-  int i, n;
-  char cQuote;
-  char *z;
-
-  if( p->zDestTable ){
-    free(p->zDestTable);
-    p->zDestTable = 0;
-  }
-  if( zName==0 ) return;
-  cQuote = quoteChar(zName);
-  n = strlen30(zName);
-  if( cQuote ) n += n+2;
-  z = p->zDestTable = malloc( n+1 );
-  if( z==0 ){
-    raw_printf(stderr,"Error: out of memory\n");
-    exit(1);
-  }
-  n = 0;
-  if( cQuote ) z[n++] = cQuote;
-  for(i=0; zName[i]; i++){
-    z[n++] = zName[i];
-    if( zName[i]==cQuote ) z[n++] = cQuote;
-  }
-  if( cQuote ) z[n++] = cQuote;
-  z[n] = 0;
-}
-
-
-/*
-** Execute a query statement that will generate SQL output.  Print
-** the result columns, comma-separated, on a line and then add a
-** semicolon terminator to the end of that line.
-**
-** If the number of columns is 1 and that column contains text "--"
-** then write the semicolon on a separate line.  That way, if a
-** "--" comment occurs at the end of the statement, the comment
-** won't consume the semicolon terminator.
-*/
-static int run_table_dump_query(
-  ShellState *p,           /* Query context */
-  const char *zSelect,     /* SELECT statement to extract content */
-  const char *zFirstRow    /* Print before first row, if not NULL */
-){
-  sqlite3_stmt *pSelect;
-  int rc;
-  int nResult;
-  int i;
-  const char *z;
-  rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0);
-  if( rc!=SQLITE_OK || !pSelect ){
-    utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc,
-                sqlite3_errmsg(p->db));
-    if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++;
-    return rc;
-  }
-  rc = sqlite3_step(pSelect);
-  nResult = sqlite3_column_count(pSelect);
-  while( rc==SQLITE_ROW ){
-    if( zFirstRow ){
-      utf8_printf(p->out, "%s", zFirstRow);
-      zFirstRow = 0;
-    }
-    z = (const char*)sqlite3_column_text(pSelect, 0);
-    utf8_printf(p->out, "%s", z);
-    for(i=1; i<nResult; i++){
-      utf8_printf(p->out, ",%s", sqlite3_column_text(pSelect, i));
-    }
-    if( z==0 ) z = "";
-    while( z[0] && (z[0]!='-' || z[1]!='-') ) z++;
-    if( z[0] ){
-      raw_printf(p->out, "\n;\n");
-    }else{
-      raw_printf(p->out, ";\n");
-    }
-    rc = sqlite3_step(pSelect);
-  }
-  rc = sqlite3_finalize(pSelect);
-  if( rc!=SQLITE_OK ){
-    utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc,
-                sqlite3_errmsg(p->db));
-    if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++;
-  }
-  return rc;
-}
-
-/*
-** Allocate space and save off current error string.
-*/
-static char *save_err_msg(
-  sqlite3 *db            /* Database to query */
-){
-  int nErrMsg = 1+strlen30(sqlite3_errmsg(db));
-  char *zErrMsg = sqlite3_malloc64(nErrMsg);
-  if( zErrMsg ){
-    memcpy(zErrMsg, sqlite3_errmsg(db), nErrMsg);
-  }
-  return zErrMsg;
-}
-
-#ifdef __linux__
-/*
-** Attempt to display I/O stats on Linux using /proc/PID/io
-*/
-static void displayLinuxIoStats(FILE *out){
-  FILE *in;
-  char z[200];
-  sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
-  in = fopen(z, "rb");
-  if( in==0 ) return;
-  while( fgets(z, sizeof(z), in)!=0 ){
-    static const struct {
-      const char *zPattern;
-      const char *zDesc;
-    } aTrans[] = {
-      { "rchar: ",                  "Bytes received by read():" },
-      { "wchar: ",                  "Bytes sent to write():"    },
-      { "syscr: ",                  "Read() system calls:"      },
-      { "syscw: ",                  "Write() system calls:"     },
-      { "read_bytes: ",             "Bytes read from storage:"  },
-      { "write_bytes: ",            "Bytes written to storage:" },
-      { "cancelled_write_bytes: ",  "Cancelled write bytes:"    },
-    };
-    int i;
-    for(i=0; i<ArraySize(aTrans); i++){
-      int n = (int)strlen(aTrans[i].zPattern);
-      if( strncmp(aTrans[i].zPattern, z, n)==0 ){
-        utf8_printf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);
-        break;
-      }
-    }
-  }
-  fclose(in);
-}
-#endif
-
-/*
-** Display a single line of status using 64-bit values.
-*/
-static void displayStatLine(
-  ShellState *p,            /* The shell context */
-  char *zLabel,             /* Label for this one line */
-  char *zFormat,            /* Format for the result */
-  int iStatusCtrl,          /* Which status to display */
-  int bReset                /* True to reset the stats */
-){
-  sqlite3_int64 iCur = -1;
-  sqlite3_int64 iHiwtr = -1;
-  int i, nPercent;
-  char zLine[200];
-  sqlite3_status64(iStatusCtrl, &iCur, &iHiwtr, bReset);
-  for(i=0, nPercent=0; zFormat[i]; i++){
-    if( zFormat[i]=='%' ) nPercent++;
-  }
-  if( nPercent>1 ){
-    sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr);
-  }else{
-    sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr);
-  }
-  raw_printf(p->out, "%-36s %s\n", zLabel, zLine);
-}
-
-/*
-** Display memory stats.
-*/
-static int display_stats(
-  sqlite3 *db,                /* Database to query */
-  ShellState *pArg,           /* Pointer to ShellState */
-  int bReset                  /* True to reset the stats */
-){
-  int iCur;
-  int iHiwtr;
-
-  if( pArg && pArg->out ){
-    displayStatLine(pArg, "Memory Used:",
-       "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset);
-    displayStatLine(pArg, "Number of Outstanding Allocations:",
-       "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset);
-    if( pArg->shellFlgs & SHFLG_Pagecache ){
-      displayStatLine(pArg, "Number of Pcache Pages Used:",
-         "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset);
-    }
-    displayStatLine(pArg, "Number of Pcache Overflow Bytes:",
-       "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset);
-    displayStatLine(pArg, "Largest Allocation:",
-       "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset);
-    displayStatLine(pArg, "Largest Pcache Allocation:",
-       "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset);
-#ifdef YYTRACKMAXSTACKDEPTH
-    displayStatLine(pArg, "Deepest Parser Stack:",
-       "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset);
-#endif
-  }
-
-  if( pArg && pArg->out && db ){
-    if( pArg->shellFlgs & SHFLG_Lookaside ){
-      iHiwtr = iCur = -1;
-      sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED,
-                        &iCur, &iHiwtr, bReset);
-      raw_printf(pArg->out,
-              "Lookaside Slots Used:                %d (max %d)\n",
-              iCur, iHiwtr);
-      sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT,
-                        &iCur, &iHiwtr, bReset);
-      raw_printf(pArg->out, "Successful lookaside attempts:       %d\n",
-              iHiwtr);
-      sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE,
-                        &iCur, &iHiwtr, bReset);
-      raw_printf(pArg->out, "Lookaside failures due to size:      %d\n",
-              iHiwtr);
-      sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL,
-                        &iCur, &iHiwtr, bReset);
-      raw_printf(pArg->out, "Lookaside failures due to OOM:       %d\n",
-              iHiwtr);
-    }
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);
-    raw_printf(pArg->out, "Pager Heap Usage:                    %d bytes\n",
-            iCur);
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);
-    raw_printf(pArg->out, "Page cache hits:                     %d\n", iCur);
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);
-    raw_printf(pArg->out, "Page cache misses:                   %d\n", iCur);
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);
-    raw_printf(pArg->out, "Page cache writes:                   %d\n", iCur);
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset);
-    raw_printf(pArg->out, "Schema Heap Usage:                   %d bytes\n",
-            iCur);
-    iHiwtr = iCur = -1;
-    sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset);
-    raw_printf(pArg->out, "Statement Heap/Lookaside Usage:      %d bytes\n",
-            iCur);
-  }
-
-  if( pArg && pArg->out && db && pArg->pStmt ){
-    iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP,
-                               bReset);
-    raw_printf(pArg->out, "Fullscan Steps:                      %d\n", iCur);
-    iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset);
-    raw_printf(pArg->out, "Sort Operations:                     %d\n", iCur);
-    iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset);
-    raw_printf(pArg->out, "Autoindex Inserts:                   %d\n", iCur);
-    iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset);
-    raw_printf(pArg->out, "Virtual Machine Steps:               %d\n", iCur);
-  }
-
-#ifdef __linux__
-  displayLinuxIoStats(pArg->out);
-#endif
-
-  /* Do not remove this machine readable comment: extra-stats-output-here */
-
-  return 0;
-}
-
-/*
-** Display scan stats.
-*/
-static void display_scanstats(
-  sqlite3 *db,                    /* Database to query */
-  ShellState *pArg                /* Pointer to ShellState */
-){
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-  UNUSED_PARAMETER(db);
-  UNUSED_PARAMETER(pArg);
-#else
-  int i, k, n, mx;
-  raw_printf(pArg->out, "-------- scanstats --------\n");
-  mx = 0;
-  for(k=0; k<=mx; k++){
-    double rEstLoop = 1.0;
-    for(i=n=0; 1; i++){
-      sqlite3_stmt *p = pArg->pStmt;
-      sqlite3_int64 nLoop, nVisit;
-      double rEst;
-      int iSid;
-      const char *zExplain;
-      if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){
-        break;
-      }
-      sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid);
-      if( iSid>mx ) mx = iSid;
-      if( iSid!=k ) continue;
-      if( n==0 ){
-        rEstLoop = (double)nLoop;
-        if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k);
-      }
-      n++;
-      sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit);
-      sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst);
-      sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain);
-      utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain);
-      rEstLoop *= rEst;
-      raw_printf(pArg->out,
-          "         nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n",
-          nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst
-      );
-    }
-  }
-  raw_printf(pArg->out, "---------------------------\n");
-#endif
-}
-
-/*
-** Parameter azArray points to a zero-terminated array of strings. zStr
-** points to a single nul-terminated string. Return non-zero if zStr
-** is equal, according to strcmp(), to any of the strings in the array.
-** Otherwise, return zero.
-*/
-static int str_in_array(const char *zStr, const char **azArray){
-  int i;
-  for(i=0; azArray[i]; i++){
-    if( 0==strcmp(zStr, azArray[i]) ) return 1;
-  }
-  return 0;
-}
-
-/*
-** If compiled statement pSql appears to be an EXPLAIN statement, allocate
-** and populate the ShellState.aiIndent[] array with the number of
-** spaces each opcode should be indented before it is output.
-**
-** The indenting rules are:
-**
-**     * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent
-**       all opcodes that occur between the p2 jump destination and the opcode
-**       itself by 2 spaces.
-**
-**     * For each "Goto", if the jump destination is earlier in the program
-**       and ends on one of:
-**          Yield  SeekGt  SeekLt  RowSetRead  Rewind
-**       or if the P1 parameter is one instead of zero,
-**       then indent all opcodes between the earlier instruction
-**       and "Goto" by 2 spaces.
-*/
-static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){
-  const char *zSql;               /* The text of the SQL statement */
-  const char *z;                  /* Used to check if this is an EXPLAIN */
-  int *abYield = 0;               /* True if op is an OP_Yield */
-  int nAlloc = 0;                 /* Allocated size of p->aiIndent[], abYield */
-  int iOp;                        /* Index of operation in p->aiIndent[] */
-
-  const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext",
-                           "NextIfOpen", "PrevIfOpen", 0 };
-  const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead",
-                            "Rewind", 0 };
-  const char *azGoto[] = { "Goto", 0 };
-
-  /* Try to figure out if this is really an EXPLAIN statement. If this
-  ** cannot be verified, return early.  */
-  if( sqlite3_column_count(pSql)!=8 ){
-    p->cMode = p->mode;
-    return;
-  }
-  zSql = sqlite3_sql(pSql);
-  if( zSql==0 ) return;
-  for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++);
-  if( sqlite3_strnicmp(z, "explain", 7) ){
-    p->cMode = p->mode;
-    return;
-  }
-
-  for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){
-    int i;
-    int iAddr = sqlite3_column_int(pSql, 0);
-    const char *zOp = (const char*)sqlite3_column_text(pSql, 1);
-
-    /* Set p2 to the P2 field of the current opcode. Then, assuming that
-    ** p2 is an instruction address, set variable p2op to the index of that
-    ** instruction in the aiIndent[] array. p2 and p2op may be different if
-    ** the current instruction is part of a sub-program generated by an
-    ** SQL trigger or foreign key.  */
-    int p2 = sqlite3_column_int(pSql, 3);
-    int p2op = (p2 + (iOp-iAddr));
-
-    /* Grow the p->aiIndent array as required */
-    if( iOp>=nAlloc ){
-      if( iOp==0 ){
-        /* Do further verfication that this is explain output.  Abort if
-        ** it is not */
-        static const char *explainCols[] = {
-           "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" };
-        int jj;
-        for(jj=0; jj<ArraySize(explainCols); jj++){
-          if( strcmp(sqlite3_column_name(pSql,jj),explainCols[jj])!=0 ){
-            p->cMode = p->mode;
-            sqlite3_reset(pSql);
-            return;
-          }
-        }
-      }
-      nAlloc += 100;
-      p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int));
-      abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int));
-    }
-    abYield[iOp] = str_in_array(zOp, azYield);
-    p->aiIndent[iOp] = 0;
-    p->nIndent = iOp+1;
-
-    if( str_in_array(zOp, azNext) ){
-      for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
-    }
-    if( str_in_array(zOp, azGoto) && p2op<p->nIndent
-     && (abYield[p2op] || sqlite3_column_int(pSql, 2))
-    ){
-      for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
-    }
-  }
-
-  p->iIndent = 0;
-  sqlite3_free(abYield);
-  sqlite3_reset(pSql);
-}
-
-/*
-** Free the array allocated by explain_data_prepare().
-*/
-static void explain_data_delete(ShellState *p){
-  sqlite3_free(p->aiIndent);
-  p->aiIndent = 0;
-  p->nIndent = 0;
-  p->iIndent = 0;
-}
-
-/*
-** Disable and restore .wheretrace and .selecttrace settings.
-*/
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE)
-extern int sqlite3SelectTrace;
-static int savedSelectTrace;
-#endif
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE)
-extern int sqlite3WhereTrace;
-static int savedWhereTrace;
-#endif
-static void disable_debug_trace_modes(void){
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE)
-  savedSelectTrace = sqlite3SelectTrace;
-  sqlite3SelectTrace = 0;
-#endif
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE)
-  savedWhereTrace = sqlite3WhereTrace;
-  sqlite3WhereTrace = 0;
-#endif
-}
-static void restore_debug_trace_modes(void){
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE)
-  sqlite3SelectTrace = savedSelectTrace;
-#endif
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE)
-  sqlite3WhereTrace = savedWhereTrace;
-#endif
-}
-
-/*
-** Run a prepared statement
-*/
-static void exec_prepared_stmt(
-  ShellState *pArg,                                /* Pointer to ShellState */
-  sqlite3_stmt *pStmt,                             /* Statment to run */
-  int (*xCallback)(void*,int,char**,char**,int*)   /* Callback function */
-){
-  int rc;
-
-  /* perform the first step.  this will tell us if we
-  ** have a result set or not and how wide it is.
-  */
-  rc = sqlite3_step(pStmt);
-  /* if we have a result set... */
-  if( SQLITE_ROW == rc ){
-    /* if we have a callback... */
-    if( xCallback ){
-      /* allocate space for col name ptr, value ptr, and type */
-      int nCol = sqlite3_column_count(pStmt);
-      void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1);
-      if( !pData ){
-        rc = SQLITE_NOMEM;
-      }else{
-        char **azCols = (char **)pData;      /* Names of result columns */
-        char **azVals = &azCols[nCol];       /* Results */
-        int *aiTypes = (int *)&azVals[nCol]; /* Result types */
-        int i, x;
-        assert(sizeof(int) <= sizeof(char *));
-        /* save off ptrs to column names */
-        for(i=0; i<nCol; i++){
-          azCols[i] = (char *)sqlite3_column_name(pStmt, i);
-        }
-        do{
-          /* extract the data and data types */
-          for(i=0; i<nCol; i++){
-            aiTypes[i] = x = sqlite3_column_type(pStmt, i);
-            if( x==SQLITE_BLOB && pArg && pArg->cMode==MODE_Insert ){
-              azVals[i] = "";
-            }else{
-              azVals[i] = (char*)sqlite3_column_text(pStmt, i);
-            }
-            if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){
-              rc = SQLITE_NOMEM;
-              break; /* from for */
-            }
-          } /* end for */
-
-          /* if data and types extracted successfully... */
-          if( SQLITE_ROW == rc ){
-            /* call the supplied callback with the result row data */
-            if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){
-              rc = SQLITE_ABORT;
-            }else{
-              rc = sqlite3_step(pStmt);
-            }
-          }
-        } while( SQLITE_ROW == rc );
-        sqlite3_free(pData);
-      }
-    }else{
-      do{
-        rc = sqlite3_step(pStmt);
-      } while( rc == SQLITE_ROW );
-    }
-  }
-}
-
-/*
-** Execute a statement or set of statements.  Print
-** any result rows/columns depending on the current mode
-** set via the supplied callback.
-**
-** This is very similar to SQLite's built-in sqlite3_exec()
-** function except it takes a slightly different callback
-** and callback data argument.
-*/
-static int shell_exec(
-  sqlite3 *db,                              /* An open database */
-  const char *zSql,                         /* SQL to be evaluated */
-  int (*xCallback)(void*,int,char**,char**,int*),   /* Callback function */
-                                            /* (not the same as sqlite3_exec) */
-  ShellState *pArg,                         /* Pointer to ShellState */
-  char **pzErrMsg                           /* Error msg written here */
-){
-  sqlite3_stmt *pStmt = NULL;     /* Statement to execute. */
-  int rc = SQLITE_OK;             /* Return Code */
-  int rc2;
-  const char *zLeftover;          /* Tail of unprocessed SQL */
-
-  if( pzErrMsg ){
-    *pzErrMsg = NULL;
-  }
-
-  while( zSql[0] && (SQLITE_OK == rc) ){
-    static const char *zStmtSql;
-    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
-    if( SQLITE_OK != rc ){
-      if( pzErrMsg ){
-        *pzErrMsg = save_err_msg(db);
-      }
-    }else{
-      if( !pStmt ){
-        /* this happens for a comment or white-space */
-        zSql = zLeftover;
-        while( IsSpace(zSql[0]) ) zSql++;
-        continue;
-      }
-      zStmtSql = sqlite3_sql(pStmt);
-      if( zStmtSql==0 ) zStmtSql = "";
-      while( IsSpace(zStmtSql[0]) ) zStmtSql++;
-
-      /* save off the prepared statment handle and reset row count */
-      if( pArg ){
-        pArg->pStmt = pStmt;
-        pArg->cnt = 0;
-      }
-
-      /* echo the sql statement if echo on */
-      if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){
-        utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql);
-      }
-
-      /* Show the EXPLAIN QUERY PLAN if .eqp is on */
-      if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){
-        sqlite3_stmt *pExplain;
-        char *zEQP;
-        disable_debug_trace_modes();
-        zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
-        rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
-        if( rc==SQLITE_OK ){
-          while( sqlite3_step(pExplain)==SQLITE_ROW ){
-            raw_printf(pArg->out,"--EQP-- %d,",sqlite3_column_int(pExplain, 0));
-            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1));
-            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2));
-            utf8_printf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3));
-          }
-        }
-        sqlite3_finalize(pExplain);
-        sqlite3_free(zEQP);
-        if( pArg->autoEQP>=2 ){
-          /* Also do an EXPLAIN for ".eqp full" mode */
-          zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
-          rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
-          if( rc==SQLITE_OK ){
-            pArg->cMode = MODE_Explain;
-            explain_data_prepare(pArg, pExplain);
-            exec_prepared_stmt(pArg, pExplain, xCallback);
-            explain_data_delete(pArg);
-          }
-          sqlite3_finalize(pExplain);
-          sqlite3_free(zEQP);
-        }
-        restore_debug_trace_modes();
-      }
-
-      if( pArg ){
-        pArg->cMode = pArg->mode;
-        if( pArg->autoExplain
-         && sqlite3_column_count(pStmt)==8
-         && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
-        ){
-          pArg->cMode = MODE_Explain;
-        }
-
-        /* If the shell is currently in ".explain" mode, gather the extra
-        ** data required to add indents to the output.*/
-        if( pArg->cMode==MODE_Explain ){
-          explain_data_prepare(pArg, pStmt);
-        }
-      }
-
-      exec_prepared_stmt(pArg, pStmt, xCallback);
-      explain_data_delete(pArg);
-
-      /* print usage stats if stats on */
-      if( pArg && pArg->statsOn ){
-        display_stats(db, pArg, 0);
-      }
-
-      /* print loop-counters if required */
-      if( pArg && pArg->scanstatsOn ){
-        display_scanstats(db, pArg);
-      }
-
-      /* Finalize the statement just executed. If this fails, save a
-      ** copy of the error message. Otherwise, set zSql to point to the
-      ** next statement to execute. */
-      rc2 = sqlite3_finalize(pStmt);
-      if( rc!=SQLITE_NOMEM ) rc = rc2;
-      if( rc==SQLITE_OK ){
-        zSql = zLeftover;
-        while( IsSpace(zSql[0]) ) zSql++;
-      }else if( pzErrMsg ){
-        *pzErrMsg = save_err_msg(db);
-      }
-
-      /* clear saved stmt handle */
-      if( pArg ){
-        pArg->pStmt = NULL;
-      }
-    }
-  } /* end while */
-
-  return rc;
-}
-
-/*
-** Release memory previously allocated by tableColumnList().
-*/
-static void freeColumnList(char **azCol){
-  int i;
-  for(i=1; azCol[i]; i++){
-    sqlite3_free(azCol[i]);
-  }
-  /* azCol[0] is a static string */
-  sqlite3_free(azCol);
-}
-
-/*
-** Return a list of pointers to strings which are the names of all
-** columns in table zTab.   The memory to hold the names is dynamically
-** allocated and must be released by the caller using a subsequent call
-** to freeColumnList().
-**
-** The azCol[0] entry is usually NULL.  However, if zTab contains a rowid
-** value that needs to be preserved, then azCol[0] is filled in with the
-** name of the rowid column.
-**
-** The first regular column in the table is azCol[1].  The list is terminated
-** by an entry with azCol[i]==0.
-*/
-static char **tableColumnList(ShellState *p, const char *zTab){
-  char **azCol = 0;
-  sqlite3_stmt *pStmt;
-  char *zSql;
-  int nCol = 0;
-  int nAlloc = 0;
-  int nPK = 0;       /* Number of PRIMARY KEY columns seen */
-  int isIPK = 0;     /* True if one PRIMARY KEY column of type INTEGER */
-  int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid);
-  int rc;
-
-  zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab);
-  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-  if( rc ) return 0;
-  while( sqlite3_step(pStmt)==SQLITE_ROW ){
-    if( nCol>=nAlloc-2 ){
-      nAlloc = nAlloc*2 + nCol + 10;
-      azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
-      if( azCol==0 ){
-        raw_printf(stderr, "Error: out of memory\n");
-        exit(1);
-      }
-    }
-    azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
-    if( sqlite3_column_int(pStmt, 5) ){
-      nPK++;
-      if( nPK==1
-       && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
-                          "INTEGER")==0
-      ){
-        isIPK = 1;
-      }else{
-        isIPK = 0;
-      }
-    }
-  }
-  sqlite3_finalize(pStmt);
-  if( azCol==0 ) return 0;
-  azCol[0] = 0;
-  azCol[nCol+1] = 0;
-
-  /* The decision of whether or not a rowid really needs to be preserved
-  ** is tricky.  We never need to preserve a rowid for a WITHOUT ROWID table
-  ** or a table with an INTEGER PRIMARY KEY.  We are unable to preserve
-  ** rowids on tables where the rowid is inaccessible because there are other
-  ** columns in the table named "rowid", "_rowid_", and "oid".
-  */
-  if( preserveRowid && isIPK ){
-    /* If a single PRIMARY KEY column with type INTEGER was seen, then it
-    ** might be an alise for the ROWID.  But it might also be a WITHOUT ROWID
-    ** table or a INTEGER PRIMARY KEY DESC column, neither of which are
-    ** ROWID aliases.  To distinguish these cases, check to see if
-    ** there is a "pk" entry in "PRAGMA index_list".  There will be
-    ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID.
-    */
-    zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)"
-                           " WHERE origin='pk'", zTab);
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-    if( rc ){
-      freeColumnList(azCol);
-      return 0;
-    }
-    rc = sqlite3_step(pStmt);
-    sqlite3_finalize(pStmt);
-    preserveRowid = rc==SQLITE_ROW;
-  }
-  if( preserveRowid ){
-    /* Only preserve the rowid if we can find a name to use for the
-    ** rowid */
-    static char *azRowid[] = { "rowid", "_rowid_", "oid" };
-    int i, j;
-    for(j=0; j<3; j++){
-      for(i=1; i<=nCol; i++){
-        if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break;
-      }
-      if( i>nCol ){
-        /* At this point, we know that azRowid[j] is not the name of any
-        ** ordinary column in the table.  Verify that azRowid[j] is a valid
-        ** name for the rowid before adding it to azCol[0].  WITHOUT ROWID
-        ** tables will fail this last check */
-        rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
-        if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
-        break;
-      }
-    }
-  }
-  return azCol;
-}
-
-/*
-** Toggle the reverse_unordered_selects setting.
-*/
-static void toggleSelectOrder(sqlite3 *db){
-  sqlite3_stmt *pStmt = 0;
-  int iSetting = 0;
-  char zStmt[100];
-  sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0);
-  if( sqlite3_step(pStmt)==SQLITE_ROW ){
-    iSetting = sqlite3_column_int(pStmt, 0);
-  }
-  sqlite3_finalize(pStmt);
-  sqlite3_snprintf(sizeof(zStmt), zStmt,
-       "PRAGMA reverse_unordered_selects(%d)", !iSetting);
-  sqlite3_exec(db, zStmt, 0, 0, 0);
-}
-
-/*
-** This is a different callback routine used for dumping the database.
-** Each row received by this callback consists of a table name,
-** the table type ("index" or "table") and SQL to create the table.
-** This routine should print text sufficient to recreate the table.
-*/
-static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
-  int rc;
-  const char *zTable;
-  const char *zType;
-  const char *zSql;
-  ShellState *p = (ShellState *)pArg;
-
-  UNUSED_PARAMETER(azNotUsed);
-  if( nArg!=3 || azArg==0 ) return 0;
-  zTable = azArg[0];
-  zType = azArg[1];
-  zSql = azArg[2];
-
-  if( strcmp(zTable, "sqlite_sequence")==0 ){
-    raw_printf(p->out, "DELETE FROM sqlite_sequence;\n");
-  }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){
-    raw_printf(p->out, "ANALYZE sqlite_master;\n");
-  }else if( strncmp(zTable, "sqlite_", 7)==0 ){
-    return 0;
-  }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
-    char *zIns;
-    if( !p->writableSchema ){
-      raw_printf(p->out, "PRAGMA writable_schema=ON;\n");
-      p->writableSchema = 1;
-    }
-    zIns = sqlite3_mprintf(
-       "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"
-       "VALUES('table','%q','%q',0,'%q');",
-       zTable, zTable, zSql);
-    utf8_printf(p->out, "%s\n", zIns);
-    sqlite3_free(zIns);
-    return 0;
-  }else{
-    printSchemaLine(p->out, zSql, ";\n");
-  }
-
-  if( strcmp(zType, "table")==0 ){
-    ShellText sSelect;
-    ShellText sTable;
-    char **azCol;
-    int i;
-    char *savedDestTable;
-    int savedMode;
-
-    azCol = tableColumnList(p, zTable);
-    if( azCol==0 ){
-      p->nErr++;
-      return 0;
-    }
-
-    /* Always quote the table name, even if it appears to be pure ascii,
-    ** in case it is a keyword. Ex:  INSERT INTO "table" ... */
-    initText(&sTable);
-    appendText(&sTable, zTable, quoteChar(zTable));
-    /* If preserving the rowid, add a column list after the table name.
-    ** In other words:  "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
-    ** instead of the usual "INSERT INTO tab VALUES(...)".
-    */
-    if( azCol[0] ){
-      appendText(&sTable, "(", 0);
-      appendText(&sTable, azCol[0], 0);
-      for(i=1; azCol[i]; i++){
-        appendText(&sTable, ",", 0);
-        appendText(&sTable, azCol[i], quoteChar(azCol[i]));
-      }
-      appendText(&sTable, ")", 0);
-    }
-
-    /* Build an appropriate SELECT statement */
-    initText(&sSelect);
-    appendText(&sSelect, "SELECT ", 0);
-    if( azCol[0] ){
-      appendText(&sSelect, azCol[0], 0);
-      appendText(&sSelect, ",", 0);
-    }
-    for(i=1; azCol[i]; i++){
-      appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
-      if( azCol[i+1] ){
-        appendText(&sSelect, ",", 0);
-      }
-    }
-    freeColumnList(azCol);
-    appendText(&sSelect, " FROM ", 0);
-    appendText(&sSelect, zTable, quoteChar(zTable));
-
-    savedDestTable = p->zDestTable;
-    savedMode = p->mode;
-    p->zDestTable = sTable.z;
-    p->mode = p->cMode = MODE_Insert;
-    rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0);
-    if( (rc&0xff)==SQLITE_CORRUPT ){
-      raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
-      toggleSelectOrder(p->db);
-      shell_exec(p->db, sSelect.z, shell_callback, p, 0);
-      toggleSelectOrder(p->db);
-    }
-    p->zDestTable = savedDestTable;
-    p->mode = savedMode;
-    freeText(&sTable);
-    freeText(&sSelect);
-    if( rc ) p->nErr++;
-  }
-  return 0;
-}
-
-/*
-** Run zQuery.  Use dump_callback() as the callback routine so that
-** the contents of the query are output as SQL statements.
-**
-** If we get a SQLITE_CORRUPT error, rerun the query after appending
-** "ORDER BY rowid DESC" to the end.
-*/
-static int run_schema_dump_query(
-  ShellState *p,
-  const char *zQuery
-){
-  int rc;
-  char *zErr = 0;
-  rc = sqlite3_exec(p->db, zQuery, dump_callback, p, &zErr);
-  if( rc==SQLITE_CORRUPT ){
-    char *zQ2;
-    int len = strlen30(zQuery);
-    raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
-    if( zErr ){
-      utf8_printf(p->out, "/****** %s ******/\n", zErr);
-      sqlite3_free(zErr);
-      zErr = 0;
-    }
-    zQ2 = malloc( len+100 );
-    if( zQ2==0 ) return rc;
-    sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery);
-    rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr);
-    if( rc ){
-      utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr);
-    }else{
-      rc = SQLITE_CORRUPT;
-    }
-    sqlite3_free(zErr);
-    free(zQ2);
-  }
-  return rc;
-}
-
-/*
-** Text of a help message
-*/
-static char zHelp[] =
-#ifndef SQLITE_OMIT_AUTHORIZATION
-  ".auth ON|OFF           Show authorizer callbacks\n"
-#endif
-  ".backup ?DB? FILE      Backup DB (default \"main\") to FILE\n"
-  ".bail on|off           Stop after hitting an error.  Default OFF\n"
-  ".binary on|off         Turn binary output on or off.  Default OFF\n"
-  ".cd DIRECTORY          Change the working directory to DIRECTORY\n"
-  ".changes on|off        Show number of rows changed by SQL\n"
-  ".check GLOB            Fail if output since .testcase does not match\n"
-  ".clone NEWDB           Clone data into NEWDB from the existing database\n"
-  ".databases             List names and files of attached databases\n"
-  ".dbinfo ?DB?           Show status information about the database\n"
-  ".dump ?TABLE? ...      Dump the database in an SQL text format\n"
-  "                         If TABLE specified, only dump tables matching\n"
-  "                         LIKE pattern TABLE.\n"
-  ".echo on|off           Turn command echo on or off\n"
-  ".eqp on|off|full       Enable or disable automatic EXPLAIN QUERY PLAN\n"
-  ".exit                  Exit this program\n"
-/* Because explain mode comes on automatically now, the ".explain" mode
-** is removed from the help screen.  It is still supported for legacy, however */
-/*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n"*/
-  ".fullschema ?--indent? Show schema and the content of sqlite_stat tables\n"
-  ".headers on|off        Turn display of headers on or off\n"
-  ".help                  Show this message\n"
-  ".import FILE TABLE     Import data from FILE into TABLE\n"
-#ifndef SQLITE_OMIT_TEST_CONTROL
-  ".imposter INDEX TABLE  Create imposter table TABLE on index INDEX\n"
-#endif
-  ".indexes ?TABLE?       Show names of all indexes\n"
-  "                         If TABLE specified, only show indexes for tables\n"
-  "                         matching LIKE pattern TABLE.\n"
-#ifdef SQLITE_ENABLE_IOTRACE
-  ".iotrace FILE          Enable I/O diagnostic logging to FILE\n"
-#endif
-  ".limit ?LIMIT? ?VAL?   Display or change the value of an SQLITE_LIMIT\n"
-  ".lint OPTIONS          Report potential schema issues. Options:\n"
-  "                         fkey-indexes     Find missing foreign key indexes\n"
-#ifndef SQLITE_OMIT_LOAD_EXTENSION
-  ".load FILE ?ENTRY?     Load an extension library\n"
-#endif
-  ".log FILE|off          Turn logging on or off.  FILE can be stderr/stdout\n"
-  ".mode MODE ?TABLE?     Set output mode where MODE is one of:\n"
-  "                         ascii    Columns/rows delimited by 0x1F and 0x1E\n"
-  "                         csv      Comma-separated values\n"
-  "                         column   Left-aligned columns.  (See .width)\n"
-  "                         html     HTML <table> code\n"
-  "                         insert   SQL insert statements for TABLE\n"
-  "                         line     One value per line\n"
-  "                         list     Values delimited by \"|\"\n"
-  "                         quote    Escape answers as for SQL\n"
-  "                         tabs     Tab-separated values\n"
-  "                         tcl      TCL list elements\n"
-  ".nullvalue STRING      Use STRING in place of NULL values\n"
-  ".once FILENAME         Output for the next SQL command only to FILENAME\n"
-  ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n"
-  "                         The --new option starts with an empty file\n"
-  ".output ?FILENAME?     Send output to FILENAME or stdout\n"
-  ".print STRING...       Print literal STRING\n"
-  ".prompt MAIN CONTINUE  Replace the standard prompts\n"
-  ".quit                  Exit this program\n"
-  ".read FILENAME         Execute SQL in FILENAME\n"
-  ".restore ?DB? FILE     Restore content of DB (default \"main\") from FILE\n"
-  ".save FILE             Write in-memory database into FILE\n"
-  ".scanstats on|off      Turn sqlite3_stmt_scanstatus() metrics on or off\n"
-  ".schema ?PATTERN?      Show the CREATE statements matching PATTERN\n"
-  "                          Add --indent for pretty-printing\n"
-  ".selftest ?--init?     Run tests defined in the SELFTEST table\n"
-  ".separator COL ?ROW?   Change the column separator and optionally the row\n"
-  "                         separator for both the output mode and .import\n"
-#if defined(SQLITE_ENABLE_SESSION)
-  ".session CMD ...       Create or control sessions\n"
-#endif
-  ".sha3sum ?OPTIONS...?  Compute a SHA3 hash of database content\n"
-  ".shell CMD ARGS...     Run CMD ARGS... in a system shell\n"
-  ".show                  Show the current values for various settings\n"
-  ".stats ?on|off?        Show stats or turn stats on or off\n"
-  ".system CMD ARGS...    Run CMD ARGS... in a system shell\n"
-  ".tables ?TABLE?        List names of tables\n"
-  "                         If TABLE specified, only list tables matching\n"
-  "                         LIKE pattern TABLE.\n"
-  ".testcase NAME         Begin redirecting output to 'testcase-out.txt'\n"
-  ".timeout MS            Try opening locked tables for MS milliseconds\n"
-  ".timer on|off          Turn SQL timer on or off\n"
-  ".trace FILE|off        Output each SQL statement as it is run\n"
-  ".vfsinfo ?AUX?         Information about the top-level VFS\n"
-  ".vfslist               List all available VFSes\n"
-  ".vfsname ?AUX?         Print the name of the VFS stack\n"
-  ".width NUM1 NUM2 ...   Set column widths for \"column\" mode\n"
-  "                         Negative values right-justify\n"
-;
-
-#if defined(SQLITE_ENABLE_SESSION)
-/*
-** Print help information for the ".sessions" command
-*/
-void session_help(ShellState *p){
-  raw_printf(p->out,
-    ".session ?NAME? SUBCOMMAND ?ARGS...?\n"
-    "If ?NAME? is omitted, the first defined session is used.\n"
-    "Subcommands:\n"
-    "   attach TABLE             Attach TABLE\n"
-    "   changeset FILE           Write a changeset into FILE\n"
-    "   close                    Close one session\n"
-    "   enable ?BOOLEAN?         Set or query the enable bit\n"
-    "   filter GLOB...           Reject tables matching GLOBs\n"
-    "   indirect ?BOOLEAN?       Mark or query the indirect status\n"
-    "   isempty                  Query whether the session is empty\n"
-    "   list                     List currently open session names\n"
-    "   open DB NAME             Open a new session on DB\n"
-    "   patchset FILE            Write a patchset into FILE\n"
-  );
-}
-#endif
-
-
-/* Forward reference */
-static int process_input(ShellState *p, FILE *in);
-
-/*
-** Read the content of file zName into memory obtained from sqlite3_malloc64()
-** and return a pointer to the buffer. The caller is responsible for freeing
-** the memory.
-**
-** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
-** read.
-**
-** For convenience, a nul-terminator byte is always appended to the data read
-** from the file before the buffer is returned. This byte is not included in
-** the final value of (*pnByte), if applicable.
-**
-** NULL is returned if any error is encountered. The final value of *pnByte
-** is undefined in this case.
-*/
-static char *readFile(const char *zName, int *pnByte){
-  FILE *in = fopen(zName, "rb");
-  long nIn;
-  size_t nRead;
-  char *pBuf;
-  if( in==0 ) return 0;
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc64( nIn+1 );
-  if( pBuf==0 ) return 0;
-  nRead = fread(pBuf, nIn, 1, in);
-  fclose(in);
-  if( nRead!=1 ){
-    sqlite3_free(pBuf);
-    return 0;
-  }
-  pBuf[nIn] = 0;
-  if( pnByte ) *pnByte = nIn;
-  return pBuf;
-}
-
-#if defined(SQLITE_ENABLE_SESSION)
-/*
-** Close a single OpenSession object and release all of its associated
-** resources.
-*/
-static void session_close(OpenSession *pSession){
-  int i;
-  sqlite3session_delete(pSession->p);
-  sqlite3_free(pSession->zName);
-  for(i=0; i<pSession->nFilter; i++){
-    sqlite3_free(pSession->azFilter[i]);
-  }
-  sqlite3_free(pSession->azFilter);
-  memset(pSession, 0, sizeof(OpenSession));
-}
-#endif
-
-/*
-** Close all OpenSession objects and release all associated resources.
-*/
-#if defined(SQLITE_ENABLE_SESSION)
-static void session_close_all(ShellState *p){
-  int i;
-  for(i=0; i<p->nSession; i++){
-    session_close(&p->aSession[i]);
-  }
-  p->nSession = 0;
-}
-#else
-# define session_close_all(X)
-#endif
-
-/*
-** Implementation of the xFilter function for an open session.  Omit
-** any tables named by ".session filter" but let all other table through.
-*/
-#if defined(SQLITE_ENABLE_SESSION)
-static int session_filter(void *pCtx, const char *zTab){
-  OpenSession *pSession = (OpenSession*)pCtx;
-  int i;
-  for(i=0; i<pSession->nFilter; i++){
-    if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0;
-  }
-  return 1;
-}
-#endif
-
-/*
-** Make sure the database is open.  If it is not, then open it.  If
-** the database fails to open, print an error message and exit.
-*/
-static void open_db(ShellState *p, int keepAlive){
-  if( p->db==0 ){
-    sqlite3_initialize();
-    sqlite3_open(p->zDbFilename, &p->db);
-    globalDb = p->db;
-    if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
-      utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n",
-          p->zDbFilename, sqlite3_errmsg(p->db));
-      if( keepAlive ) return;
-      exit(1);
-    }
-#ifndef SQLITE_OMIT_LOAD_EXTENSION
-    sqlite3_enable_load_extension(p->db, 1);
-#endif
-    sqlite3_fileio_init(p->db, 0, 0);
-    sqlite3_shathree_init(p->db, 0, 0);
-    sqlite3_completion_init(p->db, 0, 0);
-    sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
-                            shellAddSchemaName, 0, 0);
-  }
-}
-
-#if HAVE_READLINE || HAVE_EDITLINE
-/*
-** Readline completion callbacks
-*/
-static char *readline_completion_generator(const char *text, int state){
-  static sqlite3_stmt *pStmt = 0;
-  char *zRet;
-  if( state==0 ){
-    char *zSql;
-    sqlite3_finalize(pStmt);
-    zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase"
-                           "  FROM completion(%Q) ORDER BY 1", text);
-    sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-  }
-  if( sqlite3_step(pStmt)==SQLITE_ROW ){
-    zRet = strdup((const char*)sqlite3_column_text(pStmt, 0));
-  }else{
-    sqlite3_finalize(pStmt);
-    pStmt = 0;
-    zRet = 0;
-  }
-  return zRet;
-}
-static char **readline_completion(const char *zText, int iStart, int iEnd){
-  rl_attempted_completion_over = 1;
-  return rl_completion_matches(zText, readline_completion_generator);
-}
-
-#elif HAVE_LINENOISE
-/*
-** Linenoise completion callback
-*/
-static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){
-  int nLine = (int)strlen(zLine);
-  int i, iStart;
-  sqlite3_stmt *pStmt = 0;
-  char *zSql;
-  char zBuf[1000];
-
-  if( nLine>sizeof(zBuf)-30 ) return;
-  if( zLine[0]=='.' ) return;
-  for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){}
-  if( i==nLine-1 ) return;
-  iStart = i+1;
-  memcpy(zBuf, zLine, iStart);
-  zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase"
-                         "  FROM completion(%Q,%Q) ORDER BY 1",
-                         &zLine[iStart], zLine);
-  sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-  sqlite3_exec(globalDb, "PRAGMA page_count", 0, 0, 0); /* Load the schema */
-  while( sqlite3_step(pStmt)==SQLITE_ROW ){
-    const char *zCompletion = (const char*)sqlite3_column_text(pStmt, 0);
-    int nCompletion = sqlite3_column_bytes(pStmt, 0);
-    if( iStart+nCompletion < sizeof(zBuf)-1 ){
-      memcpy(zBuf+iStart, zCompletion, nCompletion+1);
-      linenoiseAddCompletion(lc, zBuf);
-    }
-  }
-  sqlite3_finalize(pStmt);
-}
-#endif
-
-/*
-** Do C-language style dequoting.
-**
-**    \a    -> alarm
-**    \b    -> backspace
-**    \t    -> tab
-**    \n    -> newline
-**    \v    -> vertical tab
-**    \f    -> form feed
-**    \r    -> carriage return
-**    \s    -> space
-**    \"    -> "
-**    \'    -> '
-**    \\    -> backslash
-**    \NNN  -> ascii character NNN in octal
-*/
-static void resolve_backslashes(char *z){
-  int i, j;
-  char c;
-  while( *z && *z!='\\' ) z++;
-  for(i=j=0; (c = z[i])!=0; i++, j++){
-    if( c=='\\' && z[i+1]!=0 ){
-      c = z[++i];
-      if( c=='a' ){
-        c = '\a';
-      }else if( c=='b' ){
-        c = '\b';
-      }else if( c=='t' ){
-        c = '\t';
-      }else if( c=='n' ){
-        c = '\n';
-      }else if( c=='v' ){
-        c = '\v';
-      }else if( c=='f' ){
-        c = '\f';
-      }else if( c=='r' ){
-        c = '\r';
-      }else if( c=='"' ){
-        c = '"';
-      }else if( c=='\'' ){
-        c = '\'';
-      }else if( c=='\\' ){
-        c = '\\';
-      }else if( c>='0' && c<='7' ){
-        c -= '0';
-        if( z[i+1]>='0' && z[i+1]<='7' ){
-          i++;
-          c = (c<<3) + z[i] - '0';
-          if( z[i+1]>='0' && z[i+1]<='7' ){
-            i++;
-            c = (c<<3) + z[i] - '0';
-          }
-        }
-      }
-    }
-    z[j] = c;
-  }
-  if( j<i ) z[j] = 0;
-}
-
-/*
-** Return the value of a hexadecimal digit.  Return -1 if the input
-** is not a hex digit.
-*/
-static int hexDigitValue(char c){
-  if( c>='0' && c<='9' ) return c - '0';
-  if( c>='a' && c<='f' ) return c - 'a' + 10;
-  if( c>='A' && c<='F' ) return c - 'A' + 10;
-  return -1;
-}
-
-/*
-** Interpret zArg as an integer value, possibly with suffixes.
-*/
-static sqlite3_int64 integerValue(const char *zArg){
-  sqlite3_int64 v = 0;
-  static const struct { char *zSuffix; int iMult; } aMult[] = {
-    { "KiB", 1024 },
-    { "MiB", 1024*1024 },
-    { "GiB", 1024*1024*1024 },
-    { "KB",  1000 },
-    { "MB",  1000000 },
-    { "GB",  1000000000 },
-    { "K",   1000 },
-    { "M",   1000000 },
-    { "G",   1000000000 },
-  };
-  int i;
-  int isNeg = 0;
-  if( zArg[0]=='-' ){
-    isNeg = 1;
-    zArg++;
-  }else if( zArg[0]=='+' ){
-    zArg++;
-  }
-  if( zArg[0]=='0' && zArg[1]=='x' ){
-    int x;
-    zArg += 2;
-    while( (x = hexDigitValue(zArg[0]))>=0 ){
-      v = (v<<4) + x;
-      zArg++;
-    }
-  }else{
-    while( IsDigit(zArg[0]) ){
-      v = v*10 + zArg[0] - '0';
-      zArg++;
-    }
-  }
-  for(i=0; i<ArraySize(aMult); i++){
-    if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
-      v *= aMult[i].iMult;
-      break;
-    }
-  }
-  return isNeg? -v : v;
-}
-
-/*
-** Interpret zArg as either an integer or a boolean value.  Return 1 or 0
-** for TRUE and FALSE.  Return the integer value if appropriate.
-*/
-static int booleanValue(const char *zArg){
-  int i;
-  if( zArg[0]=='0' && zArg[1]=='x' ){
-    for(i=2; hexDigitValue(zArg[i])>=0; i++){}
-  }else{
-    for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){}
-  }
-  if( i>0 && zArg[i]==0 ) return (int)(integerValue(zArg) & 0xffffffff);
-  if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){
-    return 1;
-  }
-  if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){
-    return 0;
-  }
-  utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n",
-          zArg);
-  return 0;
-}
-
-/*
-** Set or clear a shell flag according to a boolean value.
-*/
-static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){
-  if( booleanValue(zArg) ){
-    ShellSetFlag(p, mFlag);
-  }else{
-    ShellClearFlag(p, mFlag);
-  }
-}
-
-/*
-** Close an output file, assuming it is not stderr or stdout
-*/
-static void output_file_close(FILE *f){
-  if( f && f!=stdout && f!=stderr ) fclose(f);
-}
-
-/*
-** Try to open an output file.   The names "stdout" and "stderr" are
-** recognized and do the right thing.  NULL is returned if the output
-** filename is "off".
-*/
-static FILE *output_file_open(const char *zFile){
-  FILE *f;
-  if( strcmp(zFile,"stdout")==0 ){
-    f = stdout;
-  }else if( strcmp(zFile, "stderr")==0 ){
-    f = stderr;
-  }else if( strcmp(zFile, "off")==0 ){
-    f = 0;
-  }else{
-    f = fopen(zFile, "wb");
-    if( f==0 ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
-    }
-  }
-  return f;
-}
-
-#if !defined(SQLITE_UNTESTABLE)
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
-/*
-** A routine for handling output from sqlite3_trace().
-*/
-static int sql_trace_callback(
-  unsigned mType,
-  void *pArg,
-  void *pP,
-  void *pX
-){
-  FILE *f = (FILE*)pArg;
-  UNUSED_PARAMETER(mType);
-  UNUSED_PARAMETER(pP);
-  if( f ){
-    const char *z = (const char*)pX;
-    int i = (int)strlen(z);
-    while( i>0 && z[i-1]==';' ){ i--; }
-    utf8_printf(f, "%.*s;\n", i, z);
-  }
-  return 0;
-}
-#endif
-#endif
-
-/*
-** A no-op routine that runs with the ".breakpoint" doc-command.  This is
-** a useful spot to set a debugger breakpoint.
-*/
-static void test_breakpoint(void){
-  static int nCall = 0;
-  nCall++;
-}
-
-/*
-** An object used to read a CSV and other files for import.
-*/
-typedef struct ImportCtx ImportCtx;
-struct ImportCtx {
-  const char *zFile;  /* Name of the input file */
-  FILE *in;           /* Read the CSV text from this input stream */
-  char *z;            /* Accumulated text for a field */
-  int n;              /* Number of bytes in z */
-  int nAlloc;         /* Space allocated for z[] */
-  int nLine;          /* Current line number */
-  int bNotFirst;      /* True if one or more bytes already read */
-  int cTerm;          /* Character that terminated the most recent field */
-  int cColSep;        /* The column separator character.  (Usually ",") */
-  int cRowSep;        /* The row separator character.  (Usually "\n") */
-};
-
-/* Append a single byte to z[] */
-static void import_append_char(ImportCtx *p, int c){
-  if( p->n+1>=p->nAlloc ){
-    p->nAlloc += p->nAlloc + 100;
-    p->z = sqlite3_realloc64(p->z, p->nAlloc);
-    if( p->z==0 ){
-      raw_printf(stderr, "out of memory\n");
-      exit(1);
-    }
-  }
-  p->z[p->n++] = (char)c;
-}
-
-/* Read a single field of CSV text.  Compatible with rfc4180 and extended
-** with the option of having a separator other than ",".
-**
-**   +  Input comes from p->in.
-**   +  Store results in p->z of length p->n.  Space to hold p->z comes
-**      from sqlite3_malloc64().
-**   +  Use p->cSep as the column separator.  The default is ",".
-**   +  Use p->rSep as the row separator.  The default is "\n".
-**   +  Keep track of the line number in p->nLine.
-**   +  Store the character that terminates the field in p->cTerm.  Store
-**      EOF on end-of-file.
-**   +  Report syntax errors on stderr
-*/
-static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
-  int c;
-  int cSep = p->cColSep;
-  int rSep = p->cRowSep;
-  p->n = 0;
-  c = fgetc(p->in);
-  if( c==EOF || seenInterrupt ){
-    p->cTerm = EOF;
-    return 0;
-  }
-  if( c=='"' ){
-    int pc, ppc;
-    int startLine = p->nLine;
-    int cQuote = c;
-    pc = ppc = 0;
-    while( 1 ){
-      c = fgetc(p->in);
-      if( c==rSep ) p->nLine++;
-      if( c==cQuote ){
-        if( pc==cQuote ){
-          pc = 0;
-          continue;
-        }
-      }
-      if( (c==cSep && pc==cQuote)
-       || (c==rSep && pc==cQuote)
-       || (c==rSep && pc=='\r' && ppc==cQuote)
-       || (c==EOF && pc==cQuote)
-      ){
-        do{ p->n--; }while( p->z[p->n]!=cQuote );
-        p->cTerm = c;
-        break;
-      }
-      if( pc==cQuote && c!='\r' ){
-        utf8_printf(stderr, "%s:%d: unescaped %c character\n",
-                p->zFile, p->nLine, cQuote);
-      }
-      if( c==EOF ){
-        utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n",
-                p->zFile, startLine, cQuote);
-        p->cTerm = c;
-        break;
-      }
-      import_append_char(p, c);
-      ppc = pc;
-      pc = c;
-    }
-  }else{
-    /* If this is the first field being parsed and it begins with the
-    ** UTF-8 BOM  (0xEF BB BF) then skip the BOM */
-    if( (c&0xff)==0xef && p->bNotFirst==0 ){
-      import_append_char(p, c);
-      c = fgetc(p->in);
-      if( (c&0xff)==0xbb ){
-        import_append_char(p, c);
-        c = fgetc(p->in);
-        if( (c&0xff)==0xbf ){
-          p->bNotFirst = 1;
-          p->n = 0;
-          return csv_read_one_field(p);
-        }
-      }
-    }
-    while( c!=EOF && c!=cSep && c!=rSep ){
-      import_append_char(p, c);
-      c = fgetc(p->in);
-    }
-    if( c==rSep ){
-      p->nLine++;
-      if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--;
-    }
-    p->cTerm = c;
-  }
-  if( p->z ) p->z[p->n] = 0;
-  p->bNotFirst = 1;
-  return p->z;
-}
-
-/* Read a single field of ASCII delimited text.
-**
-**   +  Input comes from p->in.
-**   +  Store results in p->z of length p->n.  Space to hold p->z comes
-**      from sqlite3_malloc64().
-**   +  Use p->cSep as the column separator.  The default is "\x1F".
-**   +  Use p->rSep as the row separator.  The default is "\x1E".
-**   +  Keep track of the row number in p->nLine.
-**   +  Store the character that terminates the field in p->cTerm.  Store
-**      EOF on end-of-file.
-**   +  Report syntax errors on stderr
-*/
-static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){
-  int c;
-  int cSep = p->cColSep;
-  int rSep = p->cRowSep;
-  p->n = 0;
-  c = fgetc(p->in);
-  if( c==EOF || seenInterrupt ){
-    p->cTerm = EOF;
-    return 0;
-  }
-  while( c!=EOF && c!=cSep && c!=rSep ){
-    import_append_char(p, c);
-    c = fgetc(p->in);
-  }
-  if( c==rSep ){
-    p->nLine++;
-  }
-  p->cTerm = c;
-  if( p->z ) p->z[p->n] = 0;
-  return p->z;
-}
-
-/*
-** Try to transfer data for table zTable.  If an error is seen while
-** moving forward, try to go backwards.  The backwards movement won't
-** work for WITHOUT ROWID tables.
-*/
-static void tryToCloneData(
-  ShellState *p,
-  sqlite3 *newDb,
-  const char *zTable
-){
-  sqlite3_stmt *pQuery = 0;
-  sqlite3_stmt *pInsert = 0;
-  char *zQuery = 0;
-  char *zInsert = 0;
-  int rc;
-  int i, j, n;
-  int nTable = (int)strlen(zTable);
-  int k = 0;
-  int cnt = 0;
-  const int spinRate = 10000;
-
-  zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable);
-  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-  if( rc ){
-    utf8_printf(stderr, "Error %d: %s on [%s]\n",
-            sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-            zQuery);
-    goto end_data_xfer;
-  }
-  n = sqlite3_column_count(pQuery);
-  zInsert = sqlite3_malloc64(200 + nTable + n*3);
-  if( zInsert==0 ){
-    raw_printf(stderr, "out of memory\n");
-    goto end_data_xfer;
-  }
-  sqlite3_snprintf(200+nTable,zInsert,
-                   "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
-  i = (int)strlen(zInsert);
-  for(j=1; j<n; j++){
-    memcpy(zInsert+i, ",?", 2);
-    i += 2;
-  }
-  memcpy(zInsert+i, ");", 3);
-  rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0);
-  if( rc ){
-    utf8_printf(stderr, "Error %d: %s on [%s]\n",
-            sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb),
-            zQuery);
-    goto end_data_xfer;
-  }
-  for(k=0; k<2; k++){
-    while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
-      for(i=0; i<n; i++){
-        switch( sqlite3_column_type(pQuery, i) ){
-          case SQLITE_NULL: {
-            sqlite3_bind_null(pInsert, i+1);
-            break;
-          }
-          case SQLITE_INTEGER: {
-            sqlite3_bind_int64(pInsert, i+1, sqlite3_column_int64(pQuery,i));
-            break;
-          }
-          case SQLITE_FLOAT: {
-            sqlite3_bind_double(pInsert, i+1, sqlite3_column_double(pQuery,i));
-            break;
-          }
-          case SQLITE_TEXT: {
-            sqlite3_bind_text(pInsert, i+1,
-                             (const char*)sqlite3_column_text(pQuery,i),
-                             -1, SQLITE_STATIC);
-            break;
-          }
-          case SQLITE_BLOB: {
-            sqlite3_bind_blob(pInsert, i+1, sqlite3_column_blob(pQuery,i),
-                                            sqlite3_column_bytes(pQuery,i),
-                                            SQLITE_STATIC);
-            break;
-          }
-        }
-      } /* End for */
-      rc = sqlite3_step(pInsert);
-      if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
-        utf8_printf(stderr, "Error %d: %s\n", sqlite3_extended_errcode(newDb),
-                        sqlite3_errmsg(newDb));
-      }
-      sqlite3_reset(pInsert);
-      cnt++;
-      if( (cnt%spinRate)==0 ){
-        printf("%c\b", "|/-\\"[(cnt/spinRate)%4]);
-        fflush(stdout);
-      }
-    } /* End while */
-    if( rc==SQLITE_DONE ) break;
-    sqlite3_finalize(pQuery);
-    sqlite3_free(zQuery);
-    zQuery = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY rowid DESC;",
-                             zTable);
-    rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-    if( rc ){
-      utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable);
-      break;
-    }
-  } /* End for(k=0...) */
-
-end_data_xfer:
-  sqlite3_finalize(pQuery);
-  sqlite3_finalize(pInsert);
-  sqlite3_free(zQuery);
-  sqlite3_free(zInsert);
-}
-
-
-/*
-** Try to transfer all rows of the schema that match zWhere.  For
-** each row, invoke xForEach() on the object defined by that row.
-** If an error is encountered while moving forward through the
-** sqlite_master table, try again moving backwards.
-*/
-static void tryToCloneSchema(
-  ShellState *p,
-  sqlite3 *newDb,
-  const char *zWhere,
-  void (*xForEach)(ShellState*,sqlite3*,const char*)
-){
-  sqlite3_stmt *pQuery = 0;
-  char *zQuery = 0;
-  int rc;
-  const unsigned char *zName;
-  const unsigned char *zSql;
-  char *zErrMsg = 0;
-
-  zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_master"
-                           " WHERE %s", zWhere);
-  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-  if( rc ){
-    utf8_printf(stderr, "Error: (%d) %s on [%s]\n",
-                    sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-                    zQuery);
-    goto end_schema_xfer;
-  }
-  while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
-    zName = sqlite3_column_text(pQuery, 0);
-    zSql = sqlite3_column_text(pQuery, 1);
-    printf("%s... ", zName); fflush(stdout);
-    sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
-    if( zErrMsg ){
-      utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
-      sqlite3_free(zErrMsg);
-      zErrMsg = 0;
-    }
-    if( xForEach ){
-      xForEach(p, newDb, (const char*)zName);
-    }
-    printf("done\n");
-  }
-  if( rc!=SQLITE_DONE ){
-    sqlite3_finalize(pQuery);
-    sqlite3_free(zQuery);
-    zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_master"
-                             " WHERE %s ORDER BY rowid DESC", zWhere);
-    rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-    if( rc ){
-      utf8_printf(stderr, "Error: (%d) %s on [%s]\n",
-                      sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-                      zQuery);
-      goto end_schema_xfer;
-    }
-    while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
-      zName = sqlite3_column_text(pQuery, 0);
-      zSql = sqlite3_column_text(pQuery, 1);
-      printf("%s... ", zName); fflush(stdout);
-      sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
-      if( zErrMsg ){
-        utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
-        sqlite3_free(zErrMsg);
-        zErrMsg = 0;
-      }
-      if( xForEach ){
-        xForEach(p, newDb, (const char*)zName);
-      }
-      printf("done\n");
-    }
-  }
-end_schema_xfer:
-  sqlite3_finalize(pQuery);
-  sqlite3_free(zQuery);
-}
-
-/*
-** Open a new database file named "zNewDb".  Try to recover as much information
-** as possible out of the main database (which might be corrupt) and write it
-** into zNewDb.
-*/
-static void tryToClone(ShellState *p, const char *zNewDb){
-  int rc;
-  sqlite3 *newDb = 0;
-  if( access(zNewDb,0)==0 ){
-    utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb);
-    return;
-  }
-  rc = sqlite3_open(zNewDb, &newDb);
-  if( rc ){
-    utf8_printf(stderr, "Cannot create output database: %s\n",
-            sqlite3_errmsg(newDb));
-  }else{
-    sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
-    sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0);
-    tryToCloneSchema(p, newDb, "type='table'", tryToCloneData);
-    tryToCloneSchema(p, newDb, "type!='table'", 0);
-    sqlite3_exec(newDb, "COMMIT;", 0, 0, 0);
-    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
-  }
-  sqlite3_close(newDb);
-}
-
-/*
-** Change the output file back to stdout
-*/
-static void output_reset(ShellState *p){
-  if( p->outfile[0]=='|' ){
-#ifndef SQLITE_OMIT_POPEN
-    pclose(p->out);
-#endif
-  }else{
-    output_file_close(p->out);
-  }
-  p->outfile[0] = 0;
-  p->out = stdout;
-}
-
-/*
-** Run an SQL command and return the single integer result.
-*/
-static int db_int(ShellState *p, const char *zSql){
-  sqlite3_stmt *pStmt;
-  int res = 0;
-  sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-  if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
-    res = sqlite3_column_int(pStmt,0);
-  }
-  sqlite3_finalize(pStmt);
-  return res;
-}
-
-/*
-** Convert a 2-byte or 4-byte big-endian integer into a native integer
-*/
-static unsigned int get2byteInt(unsigned char *a){
-  return (a[0]<<8) + a[1];
-}
-static unsigned int get4byteInt(unsigned char *a){
-  return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3];
-}
-
-/*
-** Implementation of the ".info" command.
-**
-** Return 1 on error, 2 to exit, and 0 otherwise.
-*/
-static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
-  static const struct { const char *zName; int ofst; } aField[] = {
-     { "file change counter:",  24  },
-     { "database page count:",  28  },
-     { "freelist page count:",  36  },
-     { "schema cookie:",        40  },
-     { "schema format:",        44  },
-     { "default cache size:",   48  },
-     { "autovacuum top root:",  52  },
-     { "incremental vacuum:",   64  },
-     { "text encoding:",        56  },
-     { "user version:",         60  },
-     { "application id:",       68  },
-     { "software version:",     96  },
-  };
-  static const struct { const char *zName; const char *zSql; } aQuery[] = {
-     { "number of tables:",
-       "SELECT count(*) FROM %s WHERE type='table'" },
-     { "number of indexes:",
-       "SELECT count(*) FROM %s WHERE type='index'" },
-     { "number of triggers:",
-       "SELECT count(*) FROM %s WHERE type='trigger'" },
-     { "number of views:",
-       "SELECT count(*) FROM %s WHERE type='view'" },
-     { "schema size:",
-       "SELECT total(length(sql)) FROM %s" },
-  };
-  sqlite3_file *pFile = 0;
-  int i;
-  char *zSchemaTab;
-  char *zDb = nArg>=2 ? azArg[1] : "main";
-  unsigned char aHdr[100];
-  open_db(p, 0);
-  if( p->db==0 ) return 1;
-  sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFile);
-  if( pFile==0 || pFile->pMethods==0 || pFile->pMethods->xRead==0 ){
-    return 1;
-  }
-  i = pFile->pMethods->xRead(pFile, aHdr, 100, 0);
-  if( i!=SQLITE_OK ){
-    raw_printf(stderr, "unable to read database header\n");
-    return 1;
-  }
-  i = get2byteInt(aHdr+16);
-  if( i==1 ) i = 65536;
-  utf8_printf(p->out, "%-20s %d\n", "database page size:", i);
-  utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]);
-  utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]);
-  utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]);
-  for(i=0; i<ArraySize(aField); i++){
-    int ofst = aField[i].ofst;
-    unsigned int val = get4byteInt(aHdr + ofst);
-    utf8_printf(p->out, "%-20s %u", aField[i].zName, val);
-    switch( ofst ){
-      case 56: {
-        if( val==1 ) raw_printf(p->out, " (utf8)");
-        if( val==2 ) raw_printf(p->out, " (utf16le)");
-        if( val==3 ) raw_printf(p->out, " (utf16be)");
-      }
-    }
-    raw_printf(p->out, "\n");
-  }
-  if( zDb==0 ){
-    zSchemaTab = sqlite3_mprintf("main.sqlite_master");
-  }else if( strcmp(zDb,"temp")==0 ){
-    zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_master");
-  }else{
-    zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_master", zDb);
-  }
-  for(i=0; i<ArraySize(aQuery); i++){
-    char *zSql = sqlite3_mprintf(aQuery[i].zSql, zSchemaTab);
-    int val = db_int(p, zSql);
-    sqlite3_free(zSql);
-    utf8_printf(p->out, "%-20s %d\n", aQuery[i].zName, val);
-  }
-  sqlite3_free(zSchemaTab);
-  return 0;
-}
-
-/*
-** Print the current sqlite3_errmsg() value to stderr and return 1.
-*/
-static int shellDatabaseError(sqlite3 *db){
-  const char *zErr = sqlite3_errmsg(db);
-  utf8_printf(stderr, "Error: %s\n", zErr);
-  return 1;
-}
-
-/*
-** Print an out-of-memory message to stderr and return 1.
-*/
-static int shellNomemError(void){
-  raw_printf(stderr, "Error: out of memory\n");
-  return 1;
-}
-
-/*
-** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
-** if they match and FALSE (0) if they do not match.
-**
-** Globbing rules:
-**
-**      '*'       Matches any sequence of zero or more characters.
-**
-**      '?'       Matches exactly one character.
-**
-**     [...]      Matches one character from the enclosed list of
-**                characters.
-**
-**     [^...]     Matches one character not in the enclosed list.
-**
-**      '#'       Matches any sequence of one or more digits with an
-**                optional + or - sign in front
-**
-**      ' '       Any span of whitespace matches any other span of
-**                whitespace.
-**
-** Extra whitespace at the end of z[] is ignored.
-*/
-static int testcase_glob(const char *zGlob, const char *z){
-  int c, c2;
-  int invert;
-  int seen;
-
-  while( (c = (*(zGlob++)))!=0 ){
-    if( IsSpace(c) ){
-      if( !IsSpace(*z) ) return 0;
-      while( IsSpace(*zGlob) ) zGlob++;
-      while( IsSpace(*z) ) z++;
-    }else if( c=='*' ){
-      while( (c=(*(zGlob++))) == '*' || c=='?' ){
-        if( c=='?' && (*(z++))==0 ) return 0;
-      }
-      if( c==0 ){
-        return 1;
-      }else if( c=='[' ){
-        while( *z && testcase_glob(zGlob-1,z)==0 ){
-          z++;
-        }
-        return (*z)!=0;
-      }
-      while( (c2 = (*(z++)))!=0 ){
-        while( c2!=c ){
-          c2 = *(z++);
-          if( c2==0 ) return 0;
-        }
-        if( testcase_glob(zGlob,z) ) return 1;
-      }
-      return 0;
-    }else if( c=='?' ){
-      if( (*(z++))==0 ) return 0;
-    }else if( c=='[' ){
-      int prior_c = 0;
-      seen = 0;
-      invert = 0;
-      c = *(z++);
-      if( c==0 ) return 0;
-      c2 = *(zGlob++);
-      if( c2=='^' ){
-        invert = 1;
-        c2 = *(zGlob++);
-      }
-      if( c2==']' ){
-        if( c==']' ) seen = 1;
-        c2 = *(zGlob++);
-      }
-      while( c2 && c2!=']' ){
-        if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){
-          c2 = *(zGlob++);
-          if( c>=prior_c && c<=c2 ) seen = 1;
-          prior_c = 0;
-        }else{
-          if( c==c2 ){
-            seen = 1;
-          }
-          prior_c = c2;
-        }
-        c2 = *(zGlob++);
-      }
-      if( c2==0 || (seen ^ invert)==0 ) return 0;
-    }else if( c=='#' ){
-      if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++;
-      if( !IsDigit(z[0]) ) return 0;
-      z++;
-      while( IsDigit(z[0]) ){ z++; }
-    }else{
-      if( c!=(*(z++)) ) return 0;
-    }
-  }
-  while( IsSpace(*z) ){ z++; }
-  return *z==0;
-}
-
-
-/*
-** Compare the string as a command-line option with either one or two
-** initial "-" characters.
-*/
-static int optionMatch(const char *zStr, const char *zOpt){
-  if( zStr[0]!='-' ) return 0;
-  zStr++;
-  if( zStr[0]=='-' ) zStr++;
-  return strcmp(zStr, zOpt)==0;
-}
-
-/*
-** Delete a file.
-*/
-int shellDeleteFile(const char *zFilename){
-  int rc;
-#ifdef _WIN32
-  wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename);
-  rc = _wunlink(z);
-  sqlite3_free(z);
-#else
-  rc = unlink(zFilename);
-#endif
-  return rc;
-}
-
-
-/*
-** The implementation of SQL scalar function fkey_collate_clause(), used
-** by the ".lint fkey-indexes" command. This scalar function is always
-** called with four arguments - the parent table name, the parent column name,
-** the child table name and the child column name.
-**
-**   fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col')
-**
-** If either of the named tables or columns do not exist, this function
-** returns an empty string. An empty string is also returned if both tables
-** and columns exist but have the same default collation sequence. Or,
-** if both exist but the default collation sequences are different, this
-** function returns the string " COLLATE <parent-collation>", where
-** <parent-collation> is the default collation sequence of the parent column.
-*/
-static void shellFkeyCollateClause(
-  sqlite3_context *pCtx,
-  int nVal,
-  sqlite3_value **apVal
-){
-  sqlite3 *db = sqlite3_context_db_handle(pCtx);
-  const char *zParent;
-  const char *zParentCol;
-  const char *zParentSeq;
-  const char *zChild;
-  const char *zChildCol;
-  const char *zChildSeq = 0;  /* Initialize to avoid false-positive warning */
-  int rc;
-
-  assert( nVal==4 );
-  zParent = (const char*)sqlite3_value_text(apVal[0]);
-  zParentCol = (const char*)sqlite3_value_text(apVal[1]);
-  zChild = (const char*)sqlite3_value_text(apVal[2]);
-  zChildCol = (const char*)sqlite3_value_text(apVal[3]);
-
-  sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
-  rc = sqlite3_table_column_metadata(
-      db, "main", zParent, zParentCol, 0, &zParentSeq, 0, 0, 0
-  );
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_table_column_metadata(
-        db, "main", zChild, zChildCol, 0, &zChildSeq, 0, 0, 0
-    );
-  }
-
-  if( rc==SQLITE_OK && sqlite3_stricmp(zParentSeq, zChildSeq) ){
-    char *z = sqlite3_mprintf(" COLLATE %s", zParentSeq);
-    sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT);
-    sqlite3_free(z);
-  }
-}
-
-
-/*
-** The implementation of dot-command ".lint fkey-indexes".
-*/
-static int lintFkeyIndexes(
-  ShellState *pState,             /* Current shell tool state */
-  char **azArg,                   /* Array of arguments passed to dot command */
-  int nArg                        /* Number of entries in azArg[] */
-){
-  sqlite3 *db = pState->db;       /* Database handle to query "main" db of */
-  FILE *out = pState->out;        /* Stream to write non-error output to */
-  int bVerbose = 0;               /* If -verbose is present */
-  int bGroupByParent = 0;         /* If -groupbyparent is present */
-  int i;                          /* To iterate through azArg[] */
-  const char *zIndent = "";       /* How much to indent CREATE INDEX by */
-  int rc;                         /* Return code */
-  sqlite3_stmt *pSql = 0;         /* Compiled version of SQL statement below */
-
-  /*
-  ** This SELECT statement returns one row for each foreign key constraint
-  ** in the schema of the main database. The column values are:
-  **
-  ** 0. The text of an SQL statement similar to:
-  **
-  **      "EXPLAIN QUERY PLAN SELECT rowid FROM child_table WHERE child_key=?"
-  **
-  **    This is the same SELECT that the foreign keys implementation needs
-  **    to run internally on child tables. If there is an index that can
-  **    be used to optimize this query, then it can also be used by the FK
-  **    implementation to optimize DELETE or UPDATE statements on the parent
-  **    table.
-  **
-  ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by
-  **    the EXPLAIN QUERY PLAN command matches this pattern, then the schema
-  **    contains an index that can be used to optimize the query.
-  **
-  ** 2. Human readable text that describes the child table and columns. e.g.
-  **
-  **       "child_table(child_key1, child_key2)"
-  **
-  ** 3. Human readable text that describes the parent table and columns. e.g.
-  **
-  **       "parent_table(parent_key1, parent_key2)"
-  **
-  ** 4. A full CREATE INDEX statement for an index that could be used to
-  **    optimize DELETE or UPDATE statements on the parent table. e.g.
-  **
-  **       "CREATE INDEX child_table_child_key ON child_table(child_key)"
-  **
-  ** 5. The name of the parent table.
-  **
-  ** These six values are used by the C logic below to generate the report.
-  */
-  const char *zSql =
-  "SELECT "
-    "     'EXPLAIN QUERY PLAN SELECT rowid FROM ' || quote(s.name) || ' WHERE '"
-    "  || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' "
-    "  || fkey_collate_clause("
-    "       f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')"
-    ", "
-    "     'SEARCH TABLE ' || s.name || ' USING COVERING INDEX*('"
-    "  || group_concat('*=?', ' AND ') || ')'"
-    ", "
-    "     s.name  || '(' || group_concat(f.[from],  ', ') || ')'"
-    ", "
-    "     f.[table] || '(' || group_concat(COALESCE(f.[to], p.[name])) || ')'"
-    ", "
-    "     'CREATE INDEX ' || quote(s.name ||'_'|| group_concat(f.[from], '_'))"
-    "  || ' ON ' || quote(s.name) || '('"
-    "  || group_concat(quote(f.[from]) ||"
-    "        fkey_collate_clause("
-    "          f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]), ', ')"
-    "  || ');'"
-    ", "
-    "     f.[table] "
-    "FROM sqlite_master AS s, pragma_foreign_key_list(s.name) AS f "
-    "LEFT JOIN pragma_table_info AS p ON (pk-1=seq AND p.arg=f.[table]) "
-    "GROUP BY s.name, f.id "
-    "ORDER BY (CASE WHEN ? THEN f.[table] ELSE s.name END)"
-  ;
-  const char *zGlobIPK = "SEARCH TABLE * USING INTEGER PRIMARY KEY (rowid=?)";
-
-  for(i=2; i<nArg; i++){
-    int n = (int)strlen(azArg[i]);
-    if( n>1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){
-      bVerbose = 1;
-    }
-    else if( n>1 && sqlite3_strnicmp("-groupbyparent", azArg[i], n)==0 ){
-      bGroupByParent = 1;
-      zIndent = "    ";
-    }
-    else{
-      raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n",
-          azArg[0], azArg[1]
-      );
-      return SQLITE_ERROR;
-    }
-  }
-
-  /* Register the fkey_collate_clause() SQL function */
-  rc = sqlite3_create_function(db, "fkey_collate_clause", 4, SQLITE_UTF8,
-      0, shellFkeyCollateClause, 0, 0
-  );
-
-
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0);
-  }
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int(pSql, 1, bGroupByParent);
-  }
-
-  if( rc==SQLITE_OK ){
-    int rc2;
-    char *zPrev = 0;
-    while( SQLITE_ROW==sqlite3_step(pSql) ){
-      int res = -1;
-      sqlite3_stmt *pExplain = 0;
-      const char *zEQP = (const char*)sqlite3_column_text(pSql, 0);
-      const char *zGlob = (const char*)sqlite3_column_text(pSql, 1);
-      const char *zFrom = (const char*)sqlite3_column_text(pSql, 2);
-      const char *zTarget = (const char*)sqlite3_column_text(pSql, 3);
-      const char *zCI = (const char*)sqlite3_column_text(pSql, 4);
-      const char *zParent = (const char*)sqlite3_column_text(pSql, 5);
-
-      rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
-      if( rc!=SQLITE_OK ) break;
-      if( SQLITE_ROW==sqlite3_step(pExplain) ){
-        const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3);
-        res = (
-              0==sqlite3_strglob(zGlob, zPlan)
-           || 0==sqlite3_strglob(zGlobIPK, zPlan)
-        );
-      }
-      rc = sqlite3_finalize(pExplain);
-      if( rc!=SQLITE_OK ) break;
-
-      if( res<0 ){
-        raw_printf(stderr, "Error: internal error");
-        break;
-      }else{
-        if( bGroupByParent
-        && (bVerbose || res==0)
-        && (zPrev==0 || sqlite3_stricmp(zParent, zPrev))
-        ){
-          raw_printf(out, "-- Parent table %s\n", zParent);
-          sqlite3_free(zPrev);
-          zPrev = sqlite3_mprintf("%s", zParent);
-        }
-
-        if( res==0 ){
-          raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget);
-        }else if( bVerbose ){
-          raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n",
-              zIndent, zFrom, zTarget
-          );
-        }
-      }
-    }
-    sqlite3_free(zPrev);
-
-    if( rc!=SQLITE_OK ){
-      raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
-    }
-
-    rc2 = sqlite3_finalize(pSql);
-    if( rc==SQLITE_OK && rc2!=SQLITE_OK ){
-      rc = rc2;
-      raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
-    }
-  }else{
-    raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
-  }
-
-  return rc;
-}
-
-/*
-** Implementation of ".lint" dot command.
-*/
-static int lintDotCommand(
-  ShellState *pState,             /* Current shell tool state */
-  char **azArg,                   /* Array of arguments passed to dot command */
-  int nArg                        /* Number of entries in azArg[] */
-){
-  int n;
-  n = (nArg>=2 ? (int)strlen(azArg[1]) : 0);
-  if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage;
-  return lintFkeyIndexes(pState, azArg, nArg);
-
- usage:
-  raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]);
-  raw_printf(stderr, "Where sub-commands are:\n");
-  raw_printf(stderr, "    fkey-indexes\n");
-  return SQLITE_ERROR;
-}
-
-
-/*
-** If an input line begins with "." then invoke this routine to
-** process that line.
-**
-** Return 1 on error, 2 to exit, and 0 otherwise.
-*/
-static int do_meta_command(char *zLine, ShellState *p){
-  int h = 1;
-  int nArg = 0;
-  int n, c;
-  int rc = 0;
-  char *azArg[50];
-
-  /* Parse the input line into tokens.
-  */
-  while( zLine[h] && nArg<ArraySize(azArg) ){
-    while( IsSpace(zLine[h]) ){ h++; }
-    if( zLine[h]==0 ) break;
-    if( zLine[h]=='\'' || zLine[h]=='"' ){
-      int delim = zLine[h++];
-      azArg[nArg++] = &zLine[h];
-      while( zLine[h] && zLine[h]!=delim ){
-        if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++;
-        h++;
-      }
-      if( zLine[h]==delim ){
-        zLine[h++] = 0;
-      }
-      if( delim=='"' ) resolve_backslashes(azArg[nArg-1]);
-    }else{
-      azArg[nArg++] = &zLine[h];
-      while( zLine[h] && !IsSpace(zLine[h]) ){ h++; }
-      if( zLine[h] ) zLine[h++] = 0;
-      resolve_backslashes(azArg[nArg-1]);
-    }
-  }
-
-  /* Process the input line.
-  */
-  if( nArg==0 ) return 0; /* no tokens, no error */
-  n = strlen30(azArg[0]);
-  c = azArg[0][0];
-
-#ifndef SQLITE_OMIT_AUTHORIZATION
-  if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .auth ON|OFF\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    open_db(p, 0);
-    if( booleanValue(azArg[1]) ){
-      sqlite3_set_authorizer(p->db, shellAuth, p);
-    }else{
-      sqlite3_set_authorizer(p->db, 0, 0);
-    }
-  }else
-#endif
-
-  if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0)
-   || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
-  ){
-    const char *zDestFile = 0;
-    const char *zDb = 0;
-    sqlite3 *pDest;
-    sqlite3_backup *pBackup;
-    int j;
-    for(j=1; j<nArg; j++){
-      const char *z = azArg[j];
-      if( z[0]=='-' ){
-        while( z[0]=='-' ) z++;
-        /* No options to process at this time */
-        {
-          utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
-          return 1;
-        }
-      }else if( zDestFile==0 ){
-        zDestFile = azArg[j];
-      }else if( zDb==0 ){
-        zDb = zDestFile;
-        zDestFile = azArg[j];
-      }else{
-        raw_printf(stderr, "too many arguments to .backup\n");
-        return 1;
-      }
-    }
-    if( zDestFile==0 ){
-      raw_printf(stderr, "missing FILENAME argument on .backup\n");
-      return 1;
-    }
-    if( zDb==0 ) zDb = "main";
-    rc = sqlite3_open(zDestFile, &pDest);
-    if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
-      sqlite3_close(pDest);
-      return 1;
-    }
-    open_db(p, 0);
-    pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
-    if( pBackup==0 ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
-      sqlite3_close(pDest);
-      return 1;
-    }
-    while(  (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK ){}
-    sqlite3_backup_finish(pBackup);
-    if( rc==SQLITE_DONE ){
-      rc = 0;
-    }else{
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
-      rc = 1;
-    }
-    sqlite3_close(pDest);
-  }else
-
-  if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){
-    if( nArg==2 ){
-      bail_on_error = booleanValue(azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .bail on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){
-    if( nArg==2 ){
-      if( booleanValue(azArg[1]) ){
-        setBinaryMode(p->out, 1);
-      }else{
-        setTextMode(p->out, 1);
-      }
-    }else{
-      raw_printf(stderr, "Usage: .binary on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='c' && strcmp(azArg[0],"cd")==0 ){
-    if( nArg==2 ){
-#if defined(_WIN32) || defined(WIN32)
-      wchar_t *z = sqlite3_win32_utf8_to_unicode(azArg[1]);
-      rc = !SetCurrentDirectoryW(z);
-      sqlite3_free(z);
-#else
-      rc = chdir(azArg[1]);
-#endif
-      if( rc ){
-        utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]);
-        rc = 1;
-      }
-    }else{
-      raw_printf(stderr, "Usage: .cd DIRECTORY\n");
-      rc = 1;
-    }
-  }else
-
-  /* The undocumented ".breakpoint" command causes a call to the no-op
-  ** routine named test_breakpoint().
-  */
-  if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){
-    test_breakpoint();
-  }else
-
-  if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){
-    if( nArg==2 ){
-      setOrClearFlag(p, SHFLG_CountChanges, azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .changes on|off\n");
-      rc = 1;
-    }
-  }else
-
-  /* Cancel output redirection, if it is currently set (by .testcase)
-  ** Then read the content of the testcase-out.txt file and compare against
-  ** azArg[1].  If there are differences, report an error and exit.
-  */
-  if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){
-    char *zRes = 0;
-    output_reset(p);
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .check GLOB-PATTERN\n");
-      rc = 2;
-    }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){
-      raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n");
-      rc = 2;
-    }else if( testcase_glob(azArg[1],zRes)==0 ){
-      utf8_printf(stderr,
-                 "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
-                 p->zTestcase, azArg[1], zRes);
-      rc = 2;
-    }else{
-      utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase);
-      p->nCheck++;
-    }
-    sqlite3_free(zRes);
-  }else
-
-  if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){
-    if( nArg==2 ){
-      tryToClone(p, azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .clone FILENAME\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){
-    ShellState data;
-    char *zErrMsg = 0;
-    open_db(p, 0);
-    memcpy(&data, p, sizeof(data));
-    data.showHeader = 0;
-    data.cMode = data.mode = MODE_List;
-    sqlite3_snprintf(sizeof(data.colSeparator),data.colSeparator,": ");
-    data.cnt = 0;
-    sqlite3_exec(p->db, "SELECT name, file FROM pragma_database_list",
-                 callback, &data, &zErrMsg);
-    if( zErrMsg ){
-      utf8_printf(stderr,"Error: %s\n", zErrMsg);
-      sqlite3_free(zErrMsg);
-      rc = 1;
-    }
-  }else
-
-  if( c=='d' && strncmp(azArg[0], "dbinfo", n)==0 ){
-    rc = shell_dbinfo_command(p, nArg, azArg);
-  }else
-
-  if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
-    const char *zLike = 0;
-    int i;
-    int savedShowHeader = p->showHeader;
-    ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines);
-    for(i=1; i<nArg; i++){
-      if( azArg[i][0]=='-' ){
-        const char *z = azArg[i]+1;
-        if( z[0]=='-' ) z++;
-        if( strcmp(z,"preserve-rowids")==0 ){
-#ifdef SQLITE_OMIT_VIRTUALTABLE
-          raw_printf(stderr, "The --preserve-rowids option is not compatible"
-                             " with SQLITE_OMIT_VIRTUALTABLE\n");
-          rc = 1;
-          goto meta_command_exit;
-#else
-          ShellSetFlag(p, SHFLG_PreserveRowid);
-#endif
-        }else
-        if( strcmp(z,"newlines")==0 ){
-          ShellSetFlag(p, SHFLG_Newlines);
-        }else
-        {
-          raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
-          rc = 1;
-          goto meta_command_exit;
-        }
-      }else if( zLike ){
-        raw_printf(stderr, "Usage: .dump ?--preserve-rowids? "
-                           "?--newlines? ?LIKE-PATTERN?\n");
-        rc = 1;
-        goto meta_command_exit;
-      }else{
-        zLike = azArg[i];
-      }
-    }
-    open_db(p, 0);
-    /* When playing back a "dump", the content might appear in an order
-    ** which causes immediate foreign key constraints to be violated.
-    ** So disable foreign-key constraint enforcement to prevent problems. */
-    raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
-    raw_printf(p->out, "BEGIN TRANSACTION;\n");
-    p->writableSchema = 0;
-    p->showHeader = 0;
-    /* Set writable_schema=ON since doing so forces SQLite to initialize
-    ** as much of the schema as it can even if the sqlite_master table is
-    ** corrupt. */
-    sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0);
-    p->nErr = 0;
-    if( zLike==0 ){
-      run_schema_dump_query(p,
-        "SELECT name, type, sql FROM sqlite_master "
-        "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'"
-      );
-      run_schema_dump_query(p,
-        "SELECT name, type, sql FROM sqlite_master "
-        "WHERE name=='sqlite_sequence'"
-      );
-      run_table_dump_query(p,
-        "SELECT sql FROM sqlite_master "
-        "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0
-      );
-    }else{
-      char *zSql;
-      zSql = sqlite3_mprintf(
-        "SELECT name, type, sql FROM sqlite_master "
-        "WHERE tbl_name LIKE %Q AND type=='table'"
-        "  AND sql NOT NULL", zLike);
-      run_schema_dump_query(p,zSql);
-      sqlite3_free(zSql);
-      zSql = sqlite3_mprintf(
-        "SELECT sql FROM sqlite_master "
-        "WHERE sql NOT NULL"
-        "  AND type IN ('index','trigger','view')"
-        "  AND tbl_name LIKE %Q", zLike);
-      run_table_dump_query(p, zSql, 0);
-      sqlite3_free(zSql);
-    }
-    if( p->writableSchema ){
-      raw_printf(p->out, "PRAGMA writable_schema=OFF;\n");
-      p->writableSchema = 0;
-    }
-    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
-    sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
-    raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n");
-    p->showHeader = savedShowHeader;
-  }else
-
-  if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){
-    if( nArg==2 ){
-      setOrClearFlag(p, SHFLG_Echo, azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .echo on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){
-    if( nArg==2 ){
-      if( strcmp(azArg[1],"full")==0 ){
-        p->autoEQP = 2;
-      }else{
-        p->autoEQP = booleanValue(azArg[1]);
-      }
-    }else{
-      raw_printf(stderr, "Usage: .eqp on|off|full\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){
-    if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc);
-    rc = 2;
-  }else
-
-  /* The ".explain" command is automatic now.  It is largely pointless.  It
-  ** retained purely for backwards compatibility */
-  if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){
-    int val = 1;
-    if( nArg>=2 ){
-      if( strcmp(azArg[1],"auto")==0 ){
-        val = 99;
-      }else{
-        val =  booleanValue(azArg[1]);
-      }
-    }
-    if( val==1 && p->mode!=MODE_Explain ){
-      p->normalMode = p->mode;
-      p->mode = MODE_Explain;
-      p->autoExplain = 0;
-    }else if( val==0 ){
-      if( p->mode==MODE_Explain ) p->mode = p->normalMode;
-      p->autoExplain = 0;
-    }else if( val==99 ){
-      if( p->mode==MODE_Explain ) p->mode = p->normalMode;
-      p->autoExplain = 1;
-    }
-  }else
-
-  if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){
-    ShellState data;
-    char *zErrMsg = 0;
-    int doStats = 0;
-    memcpy(&data, p, sizeof(data));
-    data.showHeader = 0;
-    data.cMode = data.mode = MODE_Semi;
-    if( nArg==2 && optionMatch(azArg[1], "indent") ){
-      data.cMode = data.mode = MODE_Pretty;
-      nArg = 1;
-    }
-    if( nArg!=1 ){
-      raw_printf(stderr, "Usage: .fullschema ?--indent?\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    open_db(p, 0);
-    rc = sqlite3_exec(p->db,
-       "SELECT sql FROM"
-       "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
-       "     FROM sqlite_master UNION ALL"
-       "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) "
-       "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' "
-       "ORDER BY rowid",
-       callback, &data, &zErrMsg
-    );
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pStmt;
-      rc = sqlite3_prepare_v2(p->db,
-               "SELECT rowid FROM sqlite_master"
-               " WHERE name GLOB 'sqlite_stat[134]'",
-               -1, &pStmt, 0);
-      doStats = sqlite3_step(pStmt)==SQLITE_ROW;
-      sqlite3_finalize(pStmt);
-    }
-    if( doStats==0 ){
-      raw_printf(p->out, "/* No STAT tables available */\n");
-    }else{
-      raw_printf(p->out, "ANALYZE sqlite_master;\n");
-      sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_master'",
-                   callback, &data, &zErrMsg);
-      data.cMode = data.mode = MODE_Insert;
-      data.zDestTable = "sqlite_stat1";
-      shell_exec(p->db, "SELECT * FROM sqlite_stat1",
-                 shell_callback, &data,&zErrMsg);
-      data.zDestTable = "sqlite_stat3";
-      shell_exec(p->db, "SELECT * FROM sqlite_stat3",
-                 shell_callback, &data,&zErrMsg);
-      data.zDestTable = "sqlite_stat4";
-      shell_exec(p->db, "SELECT * FROM sqlite_stat4",
-                 shell_callback, &data, &zErrMsg);
-      raw_printf(p->out, "ANALYZE sqlite_master;\n");
-    }
-  }else
-
-  if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){
-    if( nArg==2 ){
-      p->showHeader = booleanValue(azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .headers on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='h' && strncmp(azArg[0], "help", n)==0 ){
-    utf8_printf(p->out, "%s", zHelp);
-  }else
-
-  if( c=='i' && strncmp(azArg[0], "import", n)==0 ){
-    char *zTable;               /* Insert data into this table */
-    char *zFile;                /* Name of file to extra content from */
-    sqlite3_stmt *pStmt = NULL; /* A statement */
-    int nCol;                   /* Number of columns in the table */
-    int nByte;                  /* Number of bytes in an SQL string */
-    int i, j;                   /* Loop counters */
-    int needCommit;             /* True to COMMIT or ROLLBACK at end */
-    int nSep;                   /* Number of bytes in p->colSeparator[] */
-    char *zSql;                 /* An SQL statement */
-    ImportCtx sCtx;             /* Reader context */
-    char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */
-    int (SQLITE_CDECL *xCloser)(FILE*);      /* Func to close file */
-
-    if( nArg!=3 ){
-      raw_printf(stderr, "Usage: .import FILE TABLE\n");
-      goto meta_command_exit;
-    }
-    zFile = azArg[1];
-    zTable = azArg[2];
-    seenInterrupt = 0;
-    memset(&sCtx, 0, sizeof(sCtx));
-    open_db(p, 0);
-    nSep = strlen30(p->colSeparator);
-    if( nSep==0 ){
-      raw_printf(stderr,
-                 "Error: non-null column separator required for import\n");
-      return 1;
-    }
-    if( nSep>1 ){
-      raw_printf(stderr, "Error: multi-character column separators not allowed"
-                      " for import\n");
-      return 1;
-    }
-    nSep = strlen30(p->rowSeparator);
-    if( nSep==0 ){
-      raw_printf(stderr, "Error: non-null row separator required for import\n");
-      return 1;
-    }
-    if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator, SEP_CrLf)==0 ){
-      /* When importing CSV (only), if the row separator is set to the
-      ** default output row separator, change it to the default input
-      ** row separator.  This avoids having to maintain different input
-      ** and output row separators. */
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
-      nSep = strlen30(p->rowSeparator);
-    }
-    if( nSep>1 ){
-      raw_printf(stderr, "Error: multi-character row separators not allowed"
-                      " for import\n");
-      return 1;
-    }
-    sCtx.zFile = zFile;
-    sCtx.nLine = 1;
-    if( sCtx.zFile[0]=='|' ){
-#ifdef SQLITE_OMIT_POPEN
-      raw_printf(stderr, "Error: pipes are not supported in this OS\n");
-      return 1;
-#else
-      sCtx.in = popen(sCtx.zFile+1, "r");
-      sCtx.zFile = "<pipe>";
-      xCloser = pclose;
-#endif
-    }else{
-      sCtx.in = fopen(sCtx.zFile, "rb");
-      xCloser = fclose;
-    }
-    if( p->mode==MODE_Ascii ){
-      xRead = ascii_read_one_field;
-    }else{
-      xRead = csv_read_one_field;
-    }
-    if( sCtx.in==0 ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
-      return 1;
-    }
-    sCtx.cColSep = p->colSeparator[0];
-    sCtx.cRowSep = p->rowSeparator[0];
-    zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
-    if( zSql==0 ){
-      raw_printf(stderr, "Error: out of memory\n");
-      xCloser(sCtx.in);
-      return 1;
-    }
-    nByte = strlen30(zSql);
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
-    if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){
-      char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
-      char cSep = '(';
-      while( xRead(&sCtx) ){
-        zCreate = sqlite3_mprintf("%z%c\n  \"%w\" TEXT", zCreate, cSep, sCtx.z);
-        cSep = ',';
-        if( sCtx.cTerm!=sCtx.cColSep ) break;
-      }
-      if( cSep=='(' ){
-        sqlite3_free(zCreate);
-        sqlite3_free(sCtx.z);
-        xCloser(sCtx.in);
-        utf8_printf(stderr,"%s: empty file\n", sCtx.zFile);
-        return 1;
-      }
-      zCreate = sqlite3_mprintf("%z\n)", zCreate);
-      rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
-      sqlite3_free(zCreate);
-      if( rc ){
-        utf8_printf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable,
-                sqlite3_errmsg(p->db));
-        sqlite3_free(sCtx.z);
-        xCloser(sCtx.in);
-        return 1;
-      }
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    }
-    sqlite3_free(zSql);
-    if( rc ){
-      if (pStmt) sqlite3_finalize(pStmt);
-      utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db));
-      xCloser(sCtx.in);
-      return 1;
-    }
-    nCol = sqlite3_column_count(pStmt);
-    sqlite3_finalize(pStmt);
-    pStmt = 0;
-    if( nCol==0 ) return 0; /* no columns, no error */
-    zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 );
-    if( zSql==0 ){
-      raw_printf(stderr, "Error: out of memory\n");
-      xCloser(sCtx.in);
-      return 1;
-    }
-    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
-    j = strlen30(zSql);
-    for(i=1; i<nCol; i++){
-      zSql[j++] = ',';
-      zSql[j++] = '?';
-    }
-    zSql[j++] = ')';
-    zSql[j] = 0;
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-    if( rc ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
-      if (pStmt) sqlite3_finalize(pStmt);
-      xCloser(sCtx.in);
-      return 1;
-    }
-    needCommit = sqlite3_get_autocommit(p->db);
-    if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
-    do{
-      int startLine = sCtx.nLine;
-      for(i=0; i<nCol; i++){
-        char *z = xRead(&sCtx);
-        /*
-        ** Did we reach end-of-file before finding any columns?
-        ** If so, stop instead of NULL filling the remaining columns.
-        */
-        if( z==0 && i==0 ) break;
-        /*
-        ** Did we reach end-of-file OR end-of-line before finding any
-        ** columns in ASCII mode?  If so, stop instead of NULL filling
-        ** the remaining columns.
-        */
-        if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break;
-        sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
-        if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){
-          utf8_printf(stderr, "%s:%d: expected %d columns but found %d - "
-                          "filling the rest with NULL\n",
-                          sCtx.zFile, startLine, nCol, i+1);
-          i += 2;
-          while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; }
-        }
-      }
-      if( sCtx.cTerm==sCtx.cColSep ){
-        do{
-          xRead(&sCtx);
-          i++;
-        }while( sCtx.cTerm==sCtx.cColSep );
-        utf8_printf(stderr, "%s:%d: expected %d columns but found %d - "
-                        "extras ignored\n",
-                        sCtx.zFile, startLine, nCol, i);
-      }
-      if( i>=nCol ){
-        sqlite3_step(pStmt);
-        rc = sqlite3_reset(pStmt);
-        if( rc!=SQLITE_OK ){
-          utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile,
-                      startLine, sqlite3_errmsg(p->db));
-        }
-      }
-    }while( sCtx.cTerm!=EOF );
-
-    xCloser(sCtx.in);
-    sqlite3_free(sCtx.z);
-    sqlite3_finalize(pStmt);
-    if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
-  }else
-
-#ifndef SQLITE_UNTESTABLE
-  if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){
-    char *zSql;
-    char *zCollist = 0;
-    sqlite3_stmt *pStmt;
-    int tnum = 0;
-    int i;
-    if( nArg!=3 ){
-      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    open_db(p, 0);
-    zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master"
-                           " WHERE name='%q' AND type='index'", azArg[1]);
-    sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-    if( sqlite3_step(pStmt)==SQLITE_ROW ){
-      tnum = sqlite3_column_int(pStmt, 0);
-    }
-    sqlite3_finalize(pStmt);
-    if( tnum==0 ){
-      utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]);
-      rc = 1;
-      goto meta_command_exit;
-    }
-    zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]);
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-    i = 0;
-    while( sqlite3_step(pStmt)==SQLITE_ROW ){
-      char zLabel[20];
-      const char *zCol = (const char*)sqlite3_column_text(pStmt,2);
-      i++;
-      if( zCol==0 ){
-        if( sqlite3_column_int(pStmt,1)==-1 ){
-          zCol = "_ROWID_";
-        }else{
-          sqlite3_snprintf(sizeof(zLabel),zLabel,"expr%d",i);
-          zCol = zLabel;
-        }
-      }
-      if( zCollist==0 ){
-        zCollist = sqlite3_mprintf("\"%w\"", zCol);
-      }else{
-        zCollist = sqlite3_mprintf("%z,\"%w\"", zCollist, zCol);
-      }
-    }
-    sqlite3_finalize(pStmt);
-    zSql = sqlite3_mprintf(
-          "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%s))WITHOUT ROWID",
-          azArg[2], zCollist, zCollist);
-    sqlite3_free(zCollist);
-    rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum);
-    if( rc==SQLITE_OK ){
-      rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
-      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0);
-      if( rc ){
-        utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
-      }else{
-        utf8_printf(stdout, "%s;\n", zSql);
-        raw_printf(stdout,
-           "WARNING: writing to an imposter table will corrupt the index!\n"
-        );
-      }
-    }else{
-      raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
-      rc = 1;
-    }
-    sqlite3_free(zSql);
-  }else
-#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */
-
-#ifdef SQLITE_ENABLE_IOTRACE
-  if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){
-    SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...);
-    if( iotrace && iotrace!=stdout ) fclose(iotrace);
-    iotrace = 0;
-    if( nArg<2 ){
-      sqlite3IoTrace = 0;
-    }else if( strcmp(azArg[1], "-")==0 ){
-      sqlite3IoTrace = iotracePrintf;
-      iotrace = stdout;
-    }else{
-      iotrace = fopen(azArg[1], "w");
-      if( iotrace==0 ){
-        utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]);
-        sqlite3IoTrace = 0;
-        rc = 1;
-      }else{
-        sqlite3IoTrace = iotracePrintf;
-      }
-    }
-  }else
-#endif
-
-  if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){
-    static const struct {
-       const char *zLimitName;   /* Name of a limit */
-       int limitCode;            /* Integer code for that limit */
-    } aLimit[] = {
-      { "length",                SQLITE_LIMIT_LENGTH                    },
-      { "sql_length",            SQLITE_LIMIT_SQL_LENGTH                },
-      { "column",                SQLITE_LIMIT_COLUMN                    },
-      { "expr_depth",            SQLITE_LIMIT_EXPR_DEPTH                },
-      { "compound_select",       SQLITE_LIMIT_COMPOUND_SELECT           },
-      { "vdbe_op",               SQLITE_LIMIT_VDBE_OP                   },
-      { "function_arg",          SQLITE_LIMIT_FUNCTION_ARG              },
-      { "attached",              SQLITE_LIMIT_ATTACHED                  },
-      { "like_pattern_length",   SQLITE_LIMIT_LIKE_PATTERN_LENGTH       },
-      { "variable_number",       SQLITE_LIMIT_VARIABLE_NUMBER           },
-      { "trigger_depth",         SQLITE_LIMIT_TRIGGER_DEPTH             },
-      { "worker_threads",        SQLITE_LIMIT_WORKER_THREADS            },
-    };
-    int i, n2;
-    open_db(p, 0);
-    if( nArg==1 ){
-      for(i=0; i<ArraySize(aLimit); i++){
-        printf("%20s %d\n", aLimit[i].zLimitName,
-               sqlite3_limit(p->db, aLimit[i].limitCode, -1));
-      }
-    }else if( nArg>3 ){
-      raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n");
-      rc = 1;
-      goto meta_command_exit;
-    }else{
-      int iLimit = -1;
-      n2 = strlen30(azArg[1]);
-      for(i=0; i<ArraySize(aLimit); i++){
-        if( sqlite3_strnicmp(aLimit[i].zLimitName, azArg[1], n2)==0 ){
-          if( iLimit<0 ){
-            iLimit = i;
-          }else{
-            utf8_printf(stderr, "ambiguous limit: \"%s\"\n", azArg[1]);
-            rc = 1;
-            goto meta_command_exit;
-          }
-        }
-      }
-      if( iLimit<0 ){
-        utf8_printf(stderr, "unknown limit: \"%s\"\n"
-                        "enter \".limits\" with no arguments for a list.\n",
-                         azArg[1]);
-        rc = 1;
-        goto meta_command_exit;
-      }
-      if( nArg==3 ){
-        sqlite3_limit(p->db, aLimit[iLimit].limitCode,
-                      (int)integerValue(azArg[2]));
-      }
-      printf("%20s %d\n", aLimit[iLimit].zLimitName,
-             sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
-    }
-  }else
-
-  if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){
-    open_db(p, 0);
-    lintDotCommand(p, azArg, nArg);
-  }else
-
-#ifndef SQLITE_OMIT_LOAD_EXTENSION
-  if( c=='l' && strncmp(azArg[0], "load", n)==0 ){
-    const char *zFile, *zProc;
-    char *zErrMsg = 0;
-    if( nArg<2 ){
-      raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    zFile = azArg[1];
-    zProc = nArg>=3 ? azArg[2] : 0;
-    open_db(p, 0);
-    rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg);
-    if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: %s\n", zErrMsg);
-      sqlite3_free(zErrMsg);
-      rc = 1;
-    }
-  }else
-#endif
-
-  if( c=='l' && strncmp(azArg[0], "log", n)==0 ){
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .log FILENAME\n");
-      rc = 1;
-    }else{
-      const char *zFile = azArg[1];
-      output_file_close(p->pLog);
-      p->pLog = output_file_open(zFile);
-    }
-  }else
-
-  if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){
-    const char *zMode = nArg>=2 ? azArg[1] : "";
-    int n2 = (int)strlen(zMode);
-    int c2 = zMode[0];
-    if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){
-      p->mode = MODE_Line;
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
-    }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){
-      p->mode = MODE_Column;
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
-    }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){
-      p->mode = MODE_List;
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column);
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
-    }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){
-      p->mode = MODE_Html;
-    }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){
-      p->mode = MODE_Tcl;
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space);
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
-    }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){
-      p->mode = MODE_Csv;
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
-    }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){
-      p->mode = MODE_List;
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab);
-    }else if( c2=='i' && strncmp(azArg[1],"insert",n2)==0 ){
-      p->mode = MODE_Insert;
-      set_table_name(p, nArg>=3 ? azArg[2] : "table");
-    }else if( c2=='q' && strncmp(azArg[1],"quote",n2)==0 ){
-      p->mode = MODE_Quote;
-    }else if( c2=='a' && strncmp(azArg[1],"ascii",n2)==0 ){
-      p->mode = MODE_Ascii;
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit);
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record);
-    }else if( nArg==1 ){
-      raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]);
-    }else{
-      raw_printf(stderr, "Error: mode should be one of: "
-         "ascii column csv html insert line list quote tabs tcl\n");
-      rc = 1;
-    }
-    p->cMode = p->mode;
-  }else
-
-  if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){
-    if( nArg==2 ){
-      sqlite3_snprintf(sizeof(p->nullValue), p->nullValue,
-                       "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .nullvalue STRING\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){
-    char *zNewFilename;  /* Name of the database file to open */
-    int iName = 1;       /* Index in azArg[] of the filename */
-    int newFlag = 0;     /* True to delete file before opening */
-    /* Close the existing database */
-    session_close_all(p);
-    sqlite3_close(p->db);
-    p->db = 0;
-    p->zDbFilename = 0;
-    sqlite3_free(p->zFreeOnClose);
-    p->zFreeOnClose = 0;
-    /* Check for command-line arguments */
-    for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
-      const char *z = azArg[iName];
-      if( optionMatch(z,"new") ){
-        newFlag = 1;
-      }else if( z[0]=='-' ){
-        utf8_printf(stderr, "unknown option: %s\n", z);
-        rc = 1;
-        goto meta_command_exit;
-      }
-    }
-    /* If a filename is specified, try to open it first */
-    zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0;
-    if( zNewFilename ){
-      if( newFlag ) shellDeleteFile(zNewFilename);
-      p->zDbFilename = zNewFilename;
-      open_db(p, 1);
-      if( p->db==0 ){
-        utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename);
-        sqlite3_free(zNewFilename);
-      }else{
-        p->zFreeOnClose = zNewFilename;
-      }
-    }
-    if( p->db==0 ){
-      /* As a fall-back open a TEMP database */
-      p->zDbFilename = 0;
-      open_db(p, 0);
-    }
-  }else
-
-  if( c=='o'
-   && (strncmp(azArg[0], "output", n)==0 || strncmp(azArg[0], "once", n)==0)
-  ){
-    const char *zFile = nArg>=2 ? azArg[1] : "stdout";
-    if( nArg>2 ){
-      utf8_printf(stderr, "Usage: .%s FILE\n", azArg[0]);
-      rc = 1;
-      goto meta_command_exit;
-    }
-    if( n>1 && strncmp(azArg[0], "once", n)==0 ){
-      if( nArg<2 ){
-        raw_printf(stderr, "Usage: .once FILE\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      p->outCount = 2;
-    }else{
-      p->outCount = 0;
-    }
-    output_reset(p);
-    if( zFile[0]=='|' ){
-#ifdef SQLITE_OMIT_POPEN
-      raw_printf(stderr, "Error: pipes are not supported in this OS\n");
-      rc = 1;
-      p->out = stdout;
-#else
-      p->out = popen(zFile + 1, "w");
-      if( p->out==0 ){
-        utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1);
-        p->out = stdout;
-        rc = 1;
-      }else{
-        sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
-      }
-#endif
-    }else{
-      p->out = output_file_open(zFile);
-      if( p->out==0 ){
-        if( strcmp(zFile,"off")!=0 ){
-          utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile);
-        }
-        p->out = stdout;
-        rc = 1;
-      } else {
-        sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
-      }
-    }
-  }else
-
-  if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){
-    int i;
-    for(i=1; i<nArg; i++){
-      if( i>1 ) raw_printf(p->out, " ");
-      utf8_printf(p->out, "%s", azArg[i]);
-    }
-    raw_printf(p->out, "\n");
-  }else
-
-  if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){
-    if( nArg >= 2) {
-      strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1);
-    }
-    if( nArg >= 3) {
-      strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1);
-    }
-  }else
-
-  if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){
-    rc = 2;
-  }else
-
-  if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){
-    FILE *alt;
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .read FILE\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    alt = fopen(azArg[1], "rb");
-    if( alt==0 ){
-      utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
-      rc = 1;
-    }else{
-      rc = process_input(p, alt);
-      fclose(alt);
-    }
-  }else
-
-  if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){
-    const char *zSrcFile;
-    const char *zDb;
-    sqlite3 *pSrc;
-    sqlite3_backup *pBackup;
-    int nTimeout = 0;
-
-    if( nArg==2 ){
-      zSrcFile = azArg[1];
-      zDb = "main";
-    }else if( nArg==3 ){
-      zSrcFile = azArg[2];
-      zDb = azArg[1];
-    }else{
-      raw_printf(stderr, "Usage: .restore ?DB? FILE\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    rc = sqlite3_open(zSrcFile, &pSrc);
-    if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile);
-      sqlite3_close(pSrc);
-      return 1;
-    }
-    open_db(p, 0);
-    pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main");
-    if( pBackup==0 ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
-      sqlite3_close(pSrc);
-      return 1;
-    }
-    while( (rc = sqlite3_backup_step(pBackup,100))==SQLITE_OK
-          || rc==SQLITE_BUSY  ){
-      if( rc==SQLITE_BUSY ){
-        if( nTimeout++ >= 3 ) break;
-        sqlite3_sleep(100);
-      }
-    }
-    sqlite3_backup_finish(pBackup);
-    if( rc==SQLITE_DONE ){
-      rc = 0;
-    }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){
-      raw_printf(stderr, "Error: source database is busy\n");
-      rc = 1;
-    }else{
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
-      rc = 1;
-    }
-    sqlite3_close(pSrc);
-  }else
-
-
-  if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
-    if( nArg==2 ){
-      p->scanstatsOn = booleanValue(azArg[1]);
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-      raw_printf(stderr, "Warning: .scanstats not available in this build.\n");
-#endif
-    }else{
-      raw_printf(stderr, "Usage: .scanstats on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){
-    ShellText sSelect;
-    ShellState data;
-    char *zErrMsg = 0;
-    const char *zDiv = 0;
-    int iSchema = 0;
-
-    open_db(p, 0);
-    memcpy(&data, p, sizeof(data));
-    data.showHeader = 0;
-    data.cMode = data.mode = MODE_Semi;
-    initText(&sSelect);
-    if( nArg>=2 && optionMatch(azArg[1], "indent") ){
-      data.cMode = data.mode = MODE_Pretty;
-      nArg--;
-      if( nArg==2 ) azArg[1] = azArg[2];
-    }
-    if( nArg==2 && azArg[1][0]!='-' ){
-      int i;
-      for(i=0; azArg[1][i]; i++) azArg[1][i] = ToLower(azArg[1][i]);
-      if( strcmp(azArg[1],"sqlite_master")==0 ){
-        char *new_argv[2], *new_colv[2];
-        new_argv[0] = "CREATE TABLE sqlite_master (\n"
-                      "  type text,\n"
-                      "  name text,\n"
-                      "  tbl_name text,\n"
-                      "  rootpage integer,\n"
-                      "  sql text\n"
-                      ")";
-        new_argv[1] = 0;
-        new_colv[0] = "sql";
-        new_colv[1] = 0;
-        callback(&data, 1, new_argv, new_colv);
-        rc = SQLITE_OK;
-      }else if( strcmp(azArg[1],"sqlite_temp_master")==0 ){
-        char *new_argv[2], *new_colv[2];
-        new_argv[0] = "CREATE TEMP TABLE sqlite_temp_master (\n"
-                      "  type text,\n"
-                      "  name text,\n"
-                      "  tbl_name text,\n"
-                      "  rootpage integer,\n"
-                      "  sql text\n"
-                      ")";
-        new_argv[1] = 0;
-        new_colv[0] = "sql";
-        new_colv[1] = 0;
-        callback(&data, 1, new_argv, new_colv);
-        rc = SQLITE_OK;
-      }else{
-        zDiv = "(";
-      }
-    }else if( nArg==1 ){
-      zDiv = "(";
-    }else{
-      raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    if( zDiv ){
-      sqlite3_stmt *pStmt = 0;
-      rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list",
-                              -1, &pStmt, 0);
-      if( rc ){
-        utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
-        sqlite3_finalize(pStmt);
-        rc = 1;
-        goto meta_command_exit;
-      }
-      appendText(&sSelect, "SELECT sql FROM", 0);
-      iSchema = 0;
-      while( sqlite3_step(pStmt)==SQLITE_ROW ){
-        const char *zDb = (const char*)sqlite3_column_text(pStmt, 0);
-        char zScNum[30];
-        sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema);
-        appendText(&sSelect, zDiv, 0);
-        zDiv = " UNION ALL ";
-        if( strcmp(zDb, "main")!=0 ){
-          appendText(&sSelect, "SELECT shell_add_schema(sql,", 0);
-          appendText(&sSelect, zDb, '"');
-          appendText(&sSelect, ") AS sql, type, tbl_name, name, rowid,", 0);
-          appendText(&sSelect, zScNum, 0);
-          appendText(&sSelect, " AS snum, ", 0);
-          appendText(&sSelect, zDb, '\'');
-          appendText(&sSelect, " AS sname FROM ", 0);
-          appendText(&sSelect, zDb, '"');
-          appendText(&sSelect, ".sqlite_master", 0);
-        }else{
-          appendText(&sSelect, "SELECT sql, type, tbl_name, name, rowid, ", 0);
-          appendText(&sSelect, zScNum, 0);
-          appendText(&sSelect, " AS snum, 'main' AS sname FROM sqlite_master",0);
-        }
-      }
-      sqlite3_finalize(pStmt);
-      appendText(&sSelect, ") WHERE ", 0);
-      if( nArg>1 ){
-        char *zQarg = sqlite3_mprintf("%Q", azArg[1]);
-        if( strchr(azArg[1], '.') ){
-          appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0);
-        }else{
-          appendText(&sSelect, "lower(tbl_name)", 0);
-        }
-        appendText(&sSelect, strchr(azArg[1], '*') ? " GLOB " : " LIKE ", 0);
-        appendText(&sSelect, zQarg, 0);
-        appendText(&sSelect, " AND ", 0);
-        sqlite3_free(zQarg);
-      }
-      appendText(&sSelect, "type!='meta' AND sql IS NOT NULL"
-                           " ORDER BY snum, rowid", 0);
-      rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
-      freeText(&sSelect);
-    }
-    if( zErrMsg ){
-      utf8_printf(stderr,"Error: %s\n", zErrMsg);
-      sqlite3_free(zErrMsg);
-      rc = 1;
-    }else if( rc != SQLITE_OK ){
-      raw_printf(stderr,"Error: querying schema information\n");
-      rc = 1;
-    }else{
-      rc = 0;
-    }
-  }else
-
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE)
-  if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){
-    sqlite3SelectTrace = (int)integerValue(azArg[1]);
-  }else
-#endif
-
-#if defined(SQLITE_ENABLE_SESSION)
-  if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){
-    OpenSession *pSession = &p->aSession[0];
-    char **azCmd = &azArg[1];
-    int iSes = 0;
-    int nCmd = nArg - 1;
-    int i;
-    if( nArg<=1 ) goto session_syntax_error;
-    open_db(p, 0);
-    if( nArg>=3 ){
-      for(iSes=0; iSes<p->nSession; iSes++){
-        if( strcmp(p->aSession[iSes].zName, azArg[1])==0 ) break;
-      }
-      if( iSes<p->nSession ){
-        pSession = &p->aSession[iSes];
-        azCmd++;
-        nCmd--;
-      }else{
-        pSession = &p->aSession[0];
-        iSes = 0;
-      }
-    }
-
-    /* .session attach TABLE
-    ** Invoke the sqlite3session_attach() interface to attach a particular
-    ** table so that it is never filtered.
-    */
-    if( strcmp(azCmd[0],"attach")==0 ){
-      if( nCmd!=2 ) goto session_syntax_error;
-      if( pSession->p==0 ){
-        session_not_open:
-        raw_printf(stderr, "ERROR: No sessions are open\n");
-      }else{
-        rc = sqlite3session_attach(pSession->p, azCmd[1]);
-        if( rc ){
-          raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc);
-          rc = 0;
-        }
-      }
-    }else
-
-    /* .session changeset FILE
-    ** .session patchset FILE
-    ** Write a changeset or patchset into a file.  The file is overwritten.
-    */
-    if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){
-      FILE *out = 0;
-      if( nCmd!=2 ) goto session_syntax_error;
-      if( pSession->p==0 ) goto session_not_open;
-      out = fopen(azCmd[1], "wb");
-      if( out==0 ){
-        utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]);
-      }else{
-        int szChng;
-        void *pChng;
-        if( azCmd[0][0]=='c' ){
-          rc = sqlite3session_changeset(pSession->p, &szChng, &pChng);
-        }else{
-          rc = sqlite3session_patchset(pSession->p, &szChng, &pChng);
-        }
-        if( rc ){
-          printf("Error: error code %d\n", rc);
-          rc = 0;
-        }
-        if( pChng
-          && fwrite(pChng, szChng, 1, out)!=1 ){
-          raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n",
-                  szChng);
-        }
-        sqlite3_free(pChng);
-        fclose(out);
-      }
-    }else
-
-    /* .session close
-    ** Close the identified session
-    */
-    if( strcmp(azCmd[0], "close")==0 ){
-      if( nCmd!=1 ) goto session_syntax_error;
-      if( p->nSession ){
-        session_close(pSession);
-        p->aSession[iSes] = p->aSession[--p->nSession];
-      }
-    }else
-
-    /* .session enable ?BOOLEAN?
-    ** Query or set the enable flag
-    */
-    if( strcmp(azCmd[0], "enable")==0 ){
-      int ii;
-      if( nCmd>2 ) goto session_syntax_error;
-      ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
-      if( p->nSession ){
-        ii = sqlite3session_enable(pSession->p, ii);
-        utf8_printf(p->out, "session %s enable flag = %d\n",
-                    pSession->zName, ii);
-      }
-    }else
-
-    /* .session filter GLOB ....
-    ** Set a list of GLOB patterns of table names to be excluded.
-    */
-    if( strcmp(azCmd[0], "filter")==0 ){
-      int ii, nByte;
-      if( nCmd<2 ) goto session_syntax_error;
-      if( p->nSession ){
-        for(ii=0; ii<pSession->nFilter; ii++){
-          sqlite3_free(pSession->azFilter[ii]);
-        }
-        sqlite3_free(pSession->azFilter);
-        nByte = sizeof(pSession->azFilter[0])*(nCmd-1);
-        pSession->azFilter = sqlite3_malloc( nByte );
-        if( pSession->azFilter==0 ){
-          raw_printf(stderr, "Error: out or memory\n");
-          exit(1);
-        }
-        for(ii=1; ii<nCmd; ii++){
-          pSession->azFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]);
-        }
-        pSession->nFilter = ii-1;
-      }
-    }else
-
-    /* .session indirect ?BOOLEAN?
-    ** Query or set the indirect flag
-    */
-    if( strcmp(azCmd[0], "indirect")==0 ){
-      int ii;
-      if( nCmd>2 ) goto session_syntax_error;
-      ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
-      if( p->nSession ){
-        ii = sqlite3session_indirect(pSession->p, ii);
-        utf8_printf(p->out, "session %s indirect flag = %d\n",
-                    pSession->zName, ii);
-      }
-    }else
-
-    /* .session isempty
-    ** Determine if the session is empty
-    */
-    if( strcmp(azCmd[0], "isempty")==0 ){
-      int ii;
-      if( nCmd!=1 ) goto session_syntax_error;
-      if( p->nSession ){
-        ii = sqlite3session_isempty(pSession->p);
-        utf8_printf(p->out, "session %s isempty flag = %d\n",
-                    pSession->zName, ii);
-      }
-    }else
-
-    /* .session list
-    ** List all currently open sessions
-    */
-    if( strcmp(azCmd[0],"list")==0 ){
-      for(i=0; i<p->nSession; i++){
-        utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName);
-      }
-    }else
-
-    /* .session open DB NAME
-    ** Open a new session called NAME on the attached database DB.
-    ** DB is normally "main".
-    */
-    if( strcmp(azCmd[0],"open")==0 ){
-      char *zName;
-      if( nCmd!=3 ) goto session_syntax_error;
-      zName = azCmd[2];
-      if( zName[0]==0 ) goto session_syntax_error;
-      for(i=0; i<p->nSession; i++){
-        if( strcmp(p->aSession[i].zName,zName)==0 ){
-          utf8_printf(stderr, "Session \"%s\" already exists\n", zName);
-          goto meta_command_exit;
-        }
-      }
-      if( p->nSession>=ArraySize(p->aSession) ){
-        raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession));
-        goto meta_command_exit;
-      }
-      pSession = &p->aSession[p->nSession];
-      rc = sqlite3session_create(p->db, azCmd[1], &pSession->p);
-      if( rc ){
-        raw_printf(stderr, "Cannot open session: error code=%d\n", rc);
-        rc = 0;
-        goto meta_command_exit;
-      }
-      pSession->nFilter = 0;
-      sqlite3session_table_filter(pSession->p, session_filter, pSession);
-      p->nSession++;
-      pSession->zName = sqlite3_mprintf("%s", zName);
-    }else
-    /* If no command name matches, show a syntax error */
-    session_syntax_error:
-    session_help(p);
-  }else
-#endif
-
-#ifdef SQLITE_DEBUG
-  /* Undocumented commands for internal testing.  Subject to change
-  ** without notice. */
-  if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){
-    if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){
-      int i, v;
-      for(i=1; i<nArg; i++){
-        v = booleanValue(azArg[i]);
-        utf8_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
-      }
-    }
-    if( strncmp(azArg[0]+9, "integer", n-9)==0 ){
-      int i; sqlite3_int64 v;
-      for(i=1; i<nArg; i++){
-        char zBuf[200];
-        v = integerValue(azArg[i]);
-        sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v);
-        utf8_printf(p->out, "%s", zBuf);
-      }
-    }
-  }else
-#endif
-
-  if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){
-    int bIsInit = 0;         /* True to initialize the SELFTEST table */
-    int bVerbose = 0;        /* Verbose output */
-    int bSelftestExists;     /* True if SELFTEST already exists */
-    int i, k;                /* Loop counters */
-    int nTest = 0;           /* Number of tests runs */
-    int nErr = 0;            /* Number of errors seen */
-    ShellText str;           /* Answer for a query */
-    sqlite3_stmt *pStmt = 0; /* Query against the SELFTEST table */
-
-    open_db(p,0);
-    for(i=1; i<nArg; i++){
-      const char *z = azArg[i];
-      if( z[0]=='-' && z[1]=='-' ) z++;
-      if( strcmp(z,"-init")==0 ){
-        bIsInit = 1;
-      }else
-      if( strcmp(z,"-v")==0 ){
-        bVerbose++;
-      }else
-      {
-        utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n",
-                    azArg[i], azArg[0]);
-        raw_printf(stderr, "Should be one of: --init -v\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-    }
-    if( sqlite3_table_column_metadata(p->db,"main","selftest",0,0,0,0,0,0)
-           != SQLITE_OK ){
-      bSelftestExists = 0;
-    }else{
-      bSelftestExists = 1;
-    }
-    if( bIsInit ){
-      createSelftestTable(p);
-      bSelftestExists = 1;
-    }
-    initText(&str);
-    appendText(&str, "x", 0);
-    for(k=bSelftestExists; k>=0; k--){
-      if( k==1 ){
-        rc = sqlite3_prepare_v2(p->db,
-            "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno",
-            -1, &pStmt, 0);
-      }else{
-        rc = sqlite3_prepare_v2(p->db,
-          "VALUES(0,'memo','Missing SELFTEST table - default checks only',''),"
-          "      (1,'run','PRAGMA integrity_check','ok')",
-          -1, &pStmt, 0);
-      }
-      if( rc ){
-        raw_printf(stderr, "Error querying the selftest table\n");
-        rc = 1;
-        sqlite3_finalize(pStmt);
-        goto meta_command_exit;
-      }
-      for(i=1; sqlite3_step(pStmt)==SQLITE_ROW; i++){
-        int tno = sqlite3_column_int(pStmt, 0);
-        const char *zOp = (const char*)sqlite3_column_text(pStmt, 1);
-        const char *zSql = (const char*)sqlite3_column_text(pStmt, 2);
-        const char *zAns = (const char*)sqlite3_column_text(pStmt, 3);
-
-        k = 0;
-        if( bVerbose>0 ){
-          char *zQuote = sqlite3_mprintf("%q", zSql);
-          printf("%d: %s %s\n", tno, zOp, zSql);
-          sqlite3_free(zQuote);
-        }
-        if( strcmp(zOp,"memo")==0 ){
-          utf8_printf(p->out, "%s\n", zSql);
-        }else
-        if( strcmp(zOp,"run")==0 ){
-          char *zErrMsg = 0;
-          str.n = 0;
-          str.z[0] = 0;
-          rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
-          nTest++;
-          if( bVerbose ){
-            utf8_printf(p->out, "Result: %s\n", str.z);
-          }
-          if( rc || zErrMsg ){
-            nErr++;
-            rc = 1;
-            utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg);
-            sqlite3_free(zErrMsg);
-          }else if( strcmp(zAns,str.z)!=0 ){
-            nErr++;
-            rc = 1;
-            utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns);
-            utf8_printf(p->out, "%d:      Got: [%s]\n", tno, str.z);
-          }
-        }else
-        {
-          utf8_printf(stderr,
-            "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
-          rc = 1;
-          break;
-        }
-      } /* End loop over rows of content from SELFTEST */
-      sqlite3_finalize(pStmt);
-    } /* End loop over k */
-    freeText(&str);
-    utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest);
-  }else
-
-  if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){
-    if( nArg<2 || nArg>3 ){
-      raw_printf(stderr, "Usage: .separator COL ?ROW?\n");
-      rc = 1;
-    }
-    if( nArg>=2 ){
-      sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator,
-                       "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]);
-    }
-    if( nArg>=3 ){
-      sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator,
-                       "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]);
-    }
-  }else
-
-  if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){
-    const char *zLike = 0;   /* Which table to checksum. 0 means everything */
-    int i;                   /* Loop counter */
-    int bSchema = 0;         /* Also hash the schema */
-    int bSeparate = 0;       /* Hash each table separately */
-    int iSize = 224;         /* Hash algorithm to use */
-    int bDebug = 0;          /* Only show the query that would have run */
-    sqlite3_stmt *pStmt;     /* For querying tables names */
-    char *zSql;              /* SQL to be run */
-    char *zSep;              /* Separator */
-    ShellText sSql;          /* Complete SQL for the query to run the hash */
-    ShellText sQuery;        /* Set of queries used to read all content */
-    open_db(p, 0);
-    for(i=1; i<nArg; i++){
-      const char *z = azArg[i];
-      if( z[0]=='-' ){
-        z++;
-        if( z[0]=='-' ) z++;
-        if( strcmp(z,"schema")==0 ){
-          bSchema = 1;
-        }else
-        if( strcmp(z,"sha3-224")==0 || strcmp(z,"sha3-256")==0
-         || strcmp(z,"sha3-384")==0 || strcmp(z,"sha3-512")==0
-        ){
-          iSize = atoi(&z[5]);
-        }else
-        if( strcmp(z,"debug")==0 ){
-          bDebug = 1;
-        }else
-        {
-          utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n",
-                      azArg[i], azArg[0]);
-          raw_printf(stderr, "Should be one of: --schema"
-                             " --sha3-224 --sha3-255 --sha3-384 --sha3-512\n");
-          rc = 1;
-          goto meta_command_exit;
-        }
-      }else if( zLike ){
-        raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n");
-        rc = 1;
-        goto meta_command_exit;
-      }else{
-        zLike = z;
-        bSeparate = 1;
-        if( sqlite3_strlike("sqlite_%", zLike, 0)==0 ) bSchema = 1;
-      }
-    }
-    if( bSchema ){
-      zSql = "SELECT lower(name) FROM sqlite_master"
-             " WHERE type='table' AND coalesce(rootpage,0)>1"
-             " UNION ALL SELECT 'sqlite_master'"
-             " ORDER BY 1 collate nocase";
-    }else{
-      zSql = "SELECT lower(name) FROM sqlite_master"
-             " WHERE type='table' AND coalesce(rootpage,0)>1"
-             " AND name NOT LIKE 'sqlite_%'"
-             " ORDER BY 1 collate nocase";
-    }
-    sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    initText(&sQuery);
-    initText(&sSql);
-    appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0);
-    zSep = "VALUES(";
-    while( SQLITE_ROW==sqlite3_step(pStmt) ){
-      const char *zTab = (const char*)sqlite3_column_text(pStmt,0);
-      if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue;
-      if( strncmp(zTab, "sqlite_",7)!=0 ){
-        appendText(&sQuery,"SELECT * FROM ", 0);
-        appendText(&sQuery,zTab,'"');
-        appendText(&sQuery," NOT INDEXED;", 0);
-      }else if( strcmp(zTab, "sqlite_master")==0 ){
-        appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_master"
-                           " ORDER BY name;", 0);
-      }else if( strcmp(zTab, "sqlite_sequence")==0 ){
-        appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence"
-                           " ORDER BY name;", 0);
-      }else if( strcmp(zTab, "sqlite_stat1")==0 ){
-        appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1"
-                           " ORDER BY tbl,idx;", 0);
-      }else if( strcmp(zTab, "sqlite_stat3")==0
-             || strcmp(zTab, "sqlite_stat4")==0 ){
-        appendText(&sQuery, "SELECT * FROM ", 0);
-        appendText(&sQuery, zTab, 0);
-        appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0);
-      }
-      appendText(&sSql, zSep, 0);
-      appendText(&sSql, sQuery.z, '\'');
-      sQuery.n = 0;
-      appendText(&sSql, ",", 0);
-      appendText(&sSql, zTab, '\'');
-      zSep = "),(";
-    }
-    sqlite3_finalize(pStmt);
-    if( bSeparate ){
-      zSql = sqlite3_mprintf(
-          "%s))"
-          " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label"
-          "   FROM [sha3sum$query]",
-          sSql.z, iSize);
-    }else{
-      zSql = sqlite3_mprintf(
-          "%s))"
-          " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash"
-          "   FROM [sha3sum$query]",
-          sSql.z, iSize);
-    }
-    freeText(&sQuery);
-    freeText(&sSql);
-    if( bDebug ){
-      utf8_printf(p->out, "%s\n", zSql);
-    }else{
-      shell_exec(p->db, zSql, shell_callback, p, 0);
-    }
-    sqlite3_free(zSql);
-  }else
-
-  if( c=='s'
-   && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0)
-  ){
-    char *zCmd;
-    int i, x;
-    if( nArg<2 ){
-      raw_printf(stderr, "Usage: .system COMMAND\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    zCmd = sqlite3_mprintf(strchr(azArg[1],' ')==0?"%s":"\"%s\"", azArg[1]);
-    for(i=2; i<nArg; i++){
-      zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"",
-                             zCmd, azArg[i]);
-    }
-    x = system(zCmd);
-    sqlite3_free(zCmd);
-    if( x ) raw_printf(stderr, "System command returns %d\n", x);
-  }else
-
-  if( c=='s' && strncmp(azArg[0], "show", n)==0 ){
-    static const char *azBool[] = { "off", "on", "full", "unk" };
-    int i;
-    if( nArg!=1 ){
-      raw_printf(stderr, "Usage: .show\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    utf8_printf(p->out, "%12.12s: %s\n","echo",
-                                  azBool[ShellHasFlag(p, SHFLG_Echo)]);
-    utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
-    utf8_printf(p->out, "%12.12s: %s\n","explain",
-         p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off");
-    utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]);
-    utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]);
-    utf8_printf(p->out, "%12.12s: ", "nullvalue");
-      output_c_string(p->out, p->nullValue);
-      raw_printf(p->out, "\n");
-    utf8_printf(p->out,"%12.12s: %s\n","output",
-            strlen30(p->outfile) ? p->outfile : "stdout");
-    utf8_printf(p->out,"%12.12s: ", "colseparator");
-      output_c_string(p->out, p->colSeparator);
-      raw_printf(p->out, "\n");
-    utf8_printf(p->out,"%12.12s: ", "rowseparator");
-      output_c_string(p->out, p->rowSeparator);
-      raw_printf(p->out, "\n");
-    utf8_printf(p->out, "%12.12s: %s\n","stats", azBool[p->statsOn!=0]);
-    utf8_printf(p->out, "%12.12s: ", "width");
-    for (i=0;i<(int)ArraySize(p->colWidth) && p->colWidth[i] != 0;i++) {
-      raw_printf(p->out, "%d ", p->colWidth[i]);
-    }
-    raw_printf(p->out, "\n");
-    utf8_printf(p->out, "%12.12s: %s\n", "filename",
-                p->zDbFilename ? p->zDbFilename : "");
-  }else
-
-  if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){
-    if( nArg==2 ){
-      p->statsOn = booleanValue(azArg[1]);
-    }else if( nArg==1 ){
-      display_stats(p->db, p, 0);
-    }else{
-      raw_printf(stderr, "Usage: .stats ?on|off?\n");
-      rc = 1;
-    }
-  }else
-
-  if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0)
-   || (c=='i' && (strncmp(azArg[0], "indices", n)==0
-                 || strncmp(azArg[0], "indexes", n)==0) )
-  ){
-    sqlite3_stmt *pStmt;
-    char **azResult;
-    int nRow, nAlloc;
-    int ii;
-    ShellText s;
-    initText(&s);
-    open_db(p, 0);
-    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
-    if( rc ) return shellDatabaseError(p->db);
-
-    if( nArg>2 && c=='i' ){
-      /* It is an historical accident that the .indexes command shows an error
-      ** when called with the wrong number of arguments whereas the .tables
-      ** command does not. */
-      raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){
-      const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1);
-      if( zDbName==0 ) continue;
-      if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0);
-      if( sqlite3_stricmp(zDbName, "main")==0 ){
-        appendText(&s, "SELECT name FROM ", 0);
-      }else{
-        appendText(&s, "SELECT ", 0);
-        appendText(&s, zDbName, '\'');
-        appendText(&s, "||'.'||name FROM ", 0);
-      }
-      appendText(&s, zDbName, '"');
-      appendText(&s, ".sqlite_master ", 0);
-      if( c=='t' ){
-        appendText(&s," WHERE type IN ('table','view')"
-                      "   AND name NOT LIKE 'sqlite_%'"
-                      "   AND name LIKE ?1", 0);
-      }else{
-        appendText(&s," WHERE type='index'"
-                      "   AND tbl_name LIKE ?1", 0);
-      }
-    }
-    rc = sqlite3_finalize(pStmt);
-    appendText(&s, " ORDER BY 1", 0);
-    rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0);
-    freeText(&s);
-    if( rc ) return shellDatabaseError(p->db);
-
-    /* Run the SQL statement prepared by the above block. Store the results
-    ** as an array of nul-terminated strings in azResult[].  */
-    nRow = nAlloc = 0;
-    azResult = 0;
-    if( nArg>1 ){
-      sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC);
-    }
-    while( sqlite3_step(pStmt)==SQLITE_ROW ){
-      if( nRow>=nAlloc ){
-        char **azNew;
-        int n2 = nAlloc*2 + 10;
-        azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
-        if( azNew==0 ){
-          rc = shellNomemError();
-          break;
-        }
-        nAlloc = n2;
-        azResult = azNew;
-      }
-      azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
-      if( 0==azResult[nRow] ){
-        rc = shellNomemError();
-        break;
-      }
-      nRow++;
-    }
-    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
-      rc = shellDatabaseError(p->db);
-    }
-
-    /* Pretty-print the contents of array azResult[] to the output */
-    if( rc==0 && nRow>0 ){
-      int len, maxlen = 0;
-      int i, j;
-      int nPrintCol, nPrintRow;
-      for(i=0; i<nRow; i++){
-        len = strlen30(azResult[i]);
-        if( len>maxlen ) maxlen = len;
-      }
-      nPrintCol = 80/(maxlen+2);
-      if( nPrintCol<1 ) nPrintCol = 1;
-      nPrintRow = (nRow + nPrintCol - 1)/nPrintCol;
-      for(i=0; i<nPrintRow; i++){
-        for(j=i; j<nRow; j+=nPrintRow){
-          char *zSp = j<nPrintRow ? "" : "  ";
-          utf8_printf(p->out, "%s%-*s", zSp, maxlen,
-                      azResult[j] ? azResult[j]:"");
-        }
-        raw_printf(p->out, "\n");
-      }
-    }
-
-    for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]);
-    sqlite3_free(azResult);
-  }else
-
-  /* Begin redirecting output to the file "testcase-out.txt" */
-  if( c=='t' && strcmp(azArg[0],"testcase")==0 ){
-    output_reset(p);
-    p->out = output_file_open("testcase-out.txt");
-    if( p->out==0 ){
-      raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n");
-    }
-    if( nArg>=2 ){
-      sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]);
-    }else{
-      sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?");
-    }
-  }else
-
-#ifndef SQLITE_UNTESTABLE
-  if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){
-    static const struct {
-       const char *zCtrlName;   /* Name of a test-control option */
-       int ctrlCode;            /* Integer code for that option */
-    } aCtrl[] = {
-      { "prng_save",             SQLITE_TESTCTRL_PRNG_SAVE              },
-      { "prng_restore",          SQLITE_TESTCTRL_PRNG_RESTORE           },
-      { "prng_reset",            SQLITE_TESTCTRL_PRNG_RESET             },
-      { "bitvec_test",           SQLITE_TESTCTRL_BITVEC_TEST            },
-      { "fault_install",         SQLITE_TESTCTRL_FAULT_INSTALL          },
-      { "benign_malloc_hooks",   SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS    },
-      { "pending_byte",          SQLITE_TESTCTRL_PENDING_BYTE           },
-      { "assert",                SQLITE_TESTCTRL_ASSERT                 },
-      { "always",                SQLITE_TESTCTRL_ALWAYS                 },
-      { "reserve",               SQLITE_TESTCTRL_RESERVE                },
-      { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
-      { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
-      { "byteorder",             SQLITE_TESTCTRL_BYTEORDER              },
-      { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
-      { "imposter",              SQLITE_TESTCTRL_IMPOSTER               },
-    };
-    int testctrl = -1;
-    int rc2 = 0;
-    int i, n2;
-    open_db(p, 0);
-
-    /* convert testctrl text option to value. allow any unique prefix
-    ** of the option name, or a numerical value. */
-    n2 = strlen30(azArg[1]);
-    for(i=0; i<ArraySize(aCtrl); i++){
-      if( strncmp(azArg[1], aCtrl[i].zCtrlName, n2)==0 ){
-        if( testctrl<0 ){
-          testctrl = aCtrl[i].ctrlCode;
-        }else{
-          utf8_printf(stderr, "ambiguous option name: \"%s\"\n", azArg[1]);
-          testctrl = -1;
-          break;
-        }
-      }
-    }
-    if( testctrl<0 ) testctrl = (int)integerValue(azArg[1]);
-    if( (testctrl<SQLITE_TESTCTRL_FIRST) || (testctrl>SQLITE_TESTCTRL_LAST) ){
-      utf8_printf(stderr,"Error: invalid testctrl option: %s\n", azArg[1]);
-    }else{
-      switch(testctrl){
-
-        /* sqlite3_test_control(int, db, int) */
-        case SQLITE_TESTCTRL_OPTIMIZATIONS:
-        case SQLITE_TESTCTRL_RESERVE:
-          if( nArg==3 ){
-            int opt = (int)strtol(azArg[2], 0, 0);
-            rc2 = sqlite3_test_control(testctrl, p->db, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single int option\n",
-                    azArg[1]);
-          }
-          break;
-
-        /* sqlite3_test_control(int) */
-        case SQLITE_TESTCTRL_PRNG_SAVE:
-        case SQLITE_TESTCTRL_PRNG_RESTORE:
-        case SQLITE_TESTCTRL_PRNG_RESET:
-        case SQLITE_TESTCTRL_BYTEORDER:
-          if( nArg==2 ){
-            rc2 = sqlite3_test_control(testctrl);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes no options\n",
-                        azArg[1]);
-          }
-          break;
-
-        /* sqlite3_test_control(int, uint) */
-        case SQLITE_TESTCTRL_PENDING_BYTE:
-          if( nArg==3 ){
-            unsigned int opt = (unsigned int)integerValue(azArg[2]);
-            rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single unsigned"
-                           " int option\n", azArg[1]);
-          }
-          break;
-
-        /* sqlite3_test_control(int, int) */
-        case SQLITE_TESTCTRL_ASSERT:
-        case SQLITE_TESTCTRL_ALWAYS:
-        case SQLITE_TESTCTRL_NEVER_CORRUPT:
-          if( nArg==3 ){
-            int opt = booleanValue(azArg[2]);
-            rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single int option\n",
-                            azArg[1]);
-          }
-          break;
-
-        /* sqlite3_test_control(int, char *) */
-#ifdef SQLITE_N_KEYWORD
-        case SQLITE_TESTCTRL_ISKEYWORD:
-          if( nArg==3 ){
-            const char *opt = azArg[2];
-            rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,
-                        "Error: testctrl %s takes a single char * option\n",
-                        azArg[1]);
-          }
-          break;
-#endif
-
-        case SQLITE_TESTCTRL_IMPOSTER:
-          if( nArg==5 ){
-            rc2 = sqlite3_test_control(testctrl, p->db,
-                          azArg[2],
-                          integerValue(azArg[3]),
-                          integerValue(azArg[4]));
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          }else{
-            raw_printf(stderr,"Usage: .testctrl imposter dbName onoff tnum\n");
-          }
-          break;
-
-        case SQLITE_TESTCTRL_BITVEC_TEST:
-        case SQLITE_TESTCTRL_FAULT_INSTALL:
-        case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS:
-        default:
-          utf8_printf(stderr,
-                      "Error: CLI support for testctrl %s not implemented\n",
-                      azArg[1]);
-          break;
-      }
-    }
-  }else
-#endif /* !defined(SQLITE_UNTESTABLE) */
-
-  if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){
-    open_db(p, 0);
-    sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0);
-  }else
-
-  if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){
-    if( nArg==2 ){
-      enableTimer = booleanValue(azArg[1]);
-      if( enableTimer && !HAS_TIMER ){
-        raw_printf(stderr, "Error: timer not available on this system.\n");
-        enableTimer = 0;
-      }
-    }else{
-      raw_printf(stderr, "Usage: .timer on|off\n");
-      rc = 1;
-    }
-  }else
-
-  if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
-    open_db(p, 0);
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .trace FILE|off\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    output_file_close(p->traceOut);
-    p->traceOut = output_file_open(azArg[1]);
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
-    if( p->traceOut==0 ){
-      sqlite3_trace_v2(p->db, 0, 0, 0);
-    }else{
-      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
-    }
-#endif
-  }else
-
-#if SQLITE_USER_AUTHENTICATION
-  if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
-    if( nArg<2 ){
-      raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    open_db(p, 0);
-    if( strcmp(azArg[1],"login")==0 ){
-      if( nArg!=4 ){
-        raw_printf(stderr, "Usage: .user login USER PASSWORD\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3],
-                                    (int)strlen(azArg[3]));
-      if( rc ){
-        utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]);
-        rc = 1;
-      }
-    }else if( strcmp(azArg[1],"add")==0 ){
-      if( nArg!=5 ){
-        raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_add(p->db, azArg[2],
-                            azArg[3], (int)strlen(azArg[3]),
-                            booleanValue(azArg[4]));
-      if( rc ){
-        raw_printf(stderr, "User-Add failed: %d\n", rc);
-        rc = 1;
-      }
-    }else if( strcmp(azArg[1],"edit")==0 ){
-      if( nArg!=5 ){
-        raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_change(p->db, azArg[2],
-                              azArg[3], (int)strlen(azArg[3]),
-                              booleanValue(azArg[4]));
-      if( rc ){
-        raw_printf(stderr, "User-Edit failed: %d\n", rc);
-        rc = 1;
-      }
-    }else if( strcmp(azArg[1],"delete")==0 ){
-      if( nArg!=3 ){
-        raw_printf(stderr, "Usage: .user delete USER\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_delete(p->db, azArg[2]);
-      if( rc ){
-        raw_printf(stderr, "User-Delete failed: %d\n", rc);
-        rc = 1;
-      }
-    }else{
-      raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-  }else
-#endif /* SQLITE_USER_AUTHENTICATION */
-
-  if( c=='v' && strncmp(azArg[0], "version", n)==0 ){
-    utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
-        sqlite3_libversion(), sqlite3_sourceid());
-  }else
-
-  if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){
-    const char *zDbName = nArg==2 ? azArg[1] : "main";
-    sqlite3_vfs *pVfs = 0;
-    if( p->db ){
-      sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs);
-      if( pVfs ){
-        utf8_printf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
-        raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
-        raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
-        raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
-      }
-    }
-  }else
-
-  if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){
-    sqlite3_vfs *pVfs;
-    sqlite3_vfs *pCurrent = 0;
-    if( p->db ){
-      sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent);
-    }
-    for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){
-      utf8_printf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
-           pVfs==pCurrent ? "  <--- CURRENT" : "");
-      raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
-      raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
-      raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
-      if( pVfs->pNext ){
-        raw_printf(p->out, "-----------------------------------\n");
-      }
-    }
-  }else
-
-  if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){
-    const char *zDbName = nArg==2 ? azArg[1] : "main";
-    char *zVfsName = 0;
-    if( p->db ){
-      sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName);
-      if( zVfsName ){
-        utf8_printf(p->out, "%s\n", zVfsName);
-        sqlite3_free(zVfsName);
-      }
-    }
-  }else
-
-#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE)
-  if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){
-    sqlite3WhereTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff;
-  }else
-#endif
-
-  if( c=='w' && strncmp(azArg[0], "width", n)==0 ){
-    int j;
-    assert( nArg<=ArraySize(azArg) );
-    for(j=1; j<nArg && j<ArraySize(p->colWidth); j++){
-      p->colWidth[j-1] = (int)integerValue(azArg[j]);
-    }
-  }else
-
-  {
-    utf8_printf(stderr, "Error: unknown command or invalid arguments: "
-      " \"%s\". Enter \".help\" for help\n", azArg[0]);
-    rc = 1;
-  }
-
-meta_command_exit:
-  if( p->outCount ){
-    p->outCount--;
-    if( p->outCount==0 ) output_reset(p);
-  }
-  return rc;
-}
-
-/*
-** Return TRUE if a semicolon occurs anywhere in the first N characters
-** of string z[].
-*/
-static int line_contains_semicolon(const char *z, int N){
-  int i;
-  for(i=0; i<N; i++){  if( z[i]==';' ) return 1; }
-  return 0;
-}
-
-/*
-** Test to see if a line consists entirely of whitespace.
-*/
-static int _all_whitespace(const char *z){
-  for(; *z; z++){
-    if( IsSpace(z[0]) ) continue;
-    if( *z=='/' && z[1]=='*' ){
-      z += 2;
-      while( *z && (*z!='*' || z[1]!='/') ){ z++; }
-      if( *z==0 ) return 0;
-      z++;
-      continue;
-    }
-    if( *z=='-' && z[1]=='-' ){
-      z += 2;
-      while( *z && *z!='\n' ){ z++; }
-      if( *z==0 ) return 1;
-      continue;
-    }
-    return 0;
-  }
-  return 1;
-}
-
-/*
-** Return TRUE if the line typed in is an SQL command terminator other
-** than a semi-colon.  The SQL Server style "go" command is understood
-** as is the Oracle "/".
-*/
-static int line_is_command_terminator(const char *zLine){
-  while( IsSpace(zLine[0]) ){ zLine++; };
-  if( zLine[0]=='/' && _all_whitespace(&zLine[1]) ){
-    return 1;  /* Oracle */
-  }
-  if( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o'
-         && _all_whitespace(&zLine[2]) ){
-    return 1;  /* SQL Server */
-  }
-  return 0;
-}
-
-/*
-** Return true if zSql is a complete SQL statement.  Return false if it
-** ends in the middle of a string literal or C-style comment.
-*/
-static int line_is_complete(char *zSql, int nSql){
-  int rc;
-  if( zSql==0 ) return 1;
-  zSql[nSql] = ';';
-  zSql[nSql+1] = 0;
-  rc = sqlite3_complete(zSql);
-  zSql[nSql] = 0;
-  return rc;
-}
-
-/*
-** Run a single line of SQL
-*/
-static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
-  int rc;
-  char *zErrMsg = 0;
-
-  open_db(p, 0);
-  if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql);
-  BEGIN_TIMER;
-  rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg);
-  END_TIMER;
-  if( rc || zErrMsg ){
-    char zPrefix[100];
-    if( in!=0 || !stdin_is_interactive ){
-      sqlite3_snprintf(sizeof(zPrefix), zPrefix,
-                       "Error: near line %d:", startline);
-    }else{
-      sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:");
-    }
-    if( zErrMsg!=0 ){
-      utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg);
-      sqlite3_free(zErrMsg);
-      zErrMsg = 0;
-    }else{
-      utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db));
-    }
-    return 1;
-  }else if( ShellHasFlag(p, SHFLG_CountChanges) ){
-    raw_printf(p->out, "changes: %3d   total_changes: %d\n",
-            sqlite3_changes(p->db), sqlite3_total_changes(p->db));
-  }
-  return 0;
-}
-
-
-/*
-** Read input from *in and process it.  If *in==0 then input
-** is interactive - the user is typing it it.  Otherwise, input
-** is coming from a file or device.  A prompt is issued and history
-** is saved only if input is interactive.  An interrupt signal will
-** cause this routine to exit immediately, unless input is interactive.
-**
-** Return the number of errors.
-*/
-static int process_input(ShellState *p, FILE *in){
-  char *zLine = 0;          /* A single input line */
-  char *zSql = 0;           /* Accumulated SQL text */
-  int nLine;                /* Length of current line */
-  int nSql = 0;             /* Bytes of zSql[] used */
-  int nAlloc = 0;           /* Allocated zSql[] space */
-  int nSqlPrior = 0;        /* Bytes of zSql[] used by prior line */
-  int rc;                   /* Error code */
-  int errCnt = 0;           /* Number of errors seen */
-  int lineno = 0;           /* Current line number */
-  int startline = 0;        /* Line number for start of current input */
-
-  while( errCnt==0 || !bail_on_error || (in==0 && stdin_is_interactive) ){
-    fflush(p->out);
-    zLine = one_input_line(in, zLine, nSql>0);
-    if( zLine==0 ){
-      /* End of input */
-      if( in==0 && stdin_is_interactive ) printf("\n");
-      break;
-    }
-    if( seenInterrupt ){
-      if( in!=0 ) break;
-      seenInterrupt = 0;
-    }
-    lineno++;
-    if( nSql==0 && _all_whitespace(zLine) ){
-      if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine);
-      continue;
-    }
-    if( zLine && zLine[0]=='.' && nSql==0 ){
-      if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine);
-      rc = do_meta_command(zLine, p);
-      if( rc==2 ){ /* exit requested */
-        break;
-      }else if( rc ){
-        errCnt++;
-      }
-      continue;
-    }
-    if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){
-      memcpy(zLine,";",2);
-    }
-    nLine = strlen30(zLine);
-    if( nSql+nLine+2>=nAlloc ){
-      nAlloc = nSql+nLine+100;
-      zSql = realloc(zSql, nAlloc);
-      if( zSql==0 ){
-        raw_printf(stderr, "Error: out of memory\n");
-        exit(1);
-      }
-    }
-    nSqlPrior = nSql;
-    if( nSql==0 ){
-      int i;
-      for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
-      assert( nAlloc>0 && zSql!=0 );
-      memcpy(zSql, zLine+i, nLine+1-i);
-      startline = lineno;
-      nSql = nLine-i;
-    }else{
-      zSql[nSql++] = '\n';
-      memcpy(zSql+nSql, zLine, nLine+1);
-      nSql += nLine;
-    }
-    if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior)
-                && sqlite3_complete(zSql) ){
-      errCnt += runOneSqlLine(p, zSql, in, startline);
-      nSql = 0;
-      if( p->outCount ){
-        output_reset(p);
-        p->outCount = 0;
-      }
-    }else if( nSql && _all_whitespace(zSql) ){
-      if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql);
-      nSql = 0;
-    }
-  }
-  if( nSql && !_all_whitespace(zSql) ){
-    runOneSqlLine(p, zSql, in, startline);
-  }
-  free(zSql);
-  free(zLine);
-  return errCnt>0;
-}
-
-/*
-** Return a pathname which is the user's home directory.  A
-** 0 return indicates an error of some kind.
-*/
-static char *find_home_dir(int clearFlag){
-  static char *home_dir = NULL;
-  if( clearFlag ){
-    free(home_dir);
-    home_dir = 0;
-    return 0;
-  }
-  if( home_dir ) return home_dir;
-
-#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \
-     && !defined(__RTP__) && !defined(_WRS_KERNEL)
-  {
-    struct passwd *pwent;
-    uid_t uid = getuid();
-    if( (pwent=getpwuid(uid)) != NULL) {
-      home_dir = pwent->pw_dir;
-    }
-  }
-#endif
-
-#if defined(_WIN32_WCE)
-  /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv()
-   */
-  home_dir = "/";
-#else
-
-#if defined(_WIN32) || defined(WIN32)
-  if (!home_dir) {
-    home_dir = getenv("USERPROFILE");
-  }
-#endif
-
-  if (!home_dir) {
-    home_dir = getenv("HOME");
-  }
-
-#if defined(_WIN32) || defined(WIN32)
-  if (!home_dir) {
-    char *zDrive, *zPath;
-    int n;
-    zDrive = getenv("HOMEDRIVE");
-    zPath = getenv("HOMEPATH");
-    if( zDrive && zPath ){
-      n = strlen30(zDrive) + strlen30(zPath) + 1;
-      home_dir = malloc( n );
-      if( home_dir==0 ) return 0;
-      sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath);
-      return home_dir;
-    }
-    home_dir = "c:\\";
-  }
-#endif
-
-#endif /* !_WIN32_WCE */
-
-  if( home_dir ){
-    int n = strlen30(home_dir) + 1;
-    char *z = malloc( n );
-    if( z ) memcpy(z, home_dir, n);
-    home_dir = z;
-  }
-
-  return home_dir;
-}
-
-/*
-** Read input from the file given by sqliterc_override.  Or if that
-** parameter is NULL, take input from ~/.sqliterc
-**
-** Returns the number of errors.
-*/
-static void process_sqliterc(
-  ShellState *p,                  /* Configuration data */
-  const char *sqliterc_override   /* Name of config file. NULL to use default */
-){
-  char *home_dir = NULL;
-  const char *sqliterc = sqliterc_override;
-  char *zBuf = 0;
-  FILE *in = NULL;
-
-  if (sqliterc == NULL) {
-    home_dir = find_home_dir(0);
-    if( home_dir==0 ){
-      raw_printf(stderr, "-- warning: cannot find home directory;"
-                      " cannot read ~/.sqliterc\n");
-      return;
-    }
-    sqlite3_initialize();
-    zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
-    sqliterc = zBuf;
-  }
-  in = fopen(sqliterc,"rb");
-  if( in ){
-    if( stdin_is_interactive ){
-      utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
-    }
-    process_input(p,in);
-    fclose(in);
-  }
-  sqlite3_free(zBuf);
-}
-
-/*
-** Show available command line options
-*/
-static const char zOptions[] =
-  "   -ascii               set output mode to 'ascii'\n"
-  "   -bail                stop after hitting an error\n"
-  "   -batch               force batch I/O\n"
-  "   -column              set output mode to 'column'\n"
-  "   -cmd COMMAND         run \"COMMAND\" before reading stdin\n"
-  "   -csv                 set output mode to 'csv'\n"
-  "   -echo                print commands before execution\n"
-  "   -init FILENAME       read/process named file\n"
-  "   -[no]header          turn headers on or off\n"
-#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
-  "   -heap SIZE           Size of heap for memsys3 or memsys5\n"
-#endif
-  "   -help                show this message\n"
-  "   -html                set output mode to HTML\n"
-  "   -interactive         force interactive I/O\n"
-  "   -line                set output mode to 'line'\n"
-  "   -list                set output mode to 'list'\n"
-  "   -lookaside SIZE N    use N entries of SZ bytes for lookaside memory\n"
-  "   -mmap N              default mmap size set to N\n"
-#ifdef SQLITE_ENABLE_MULTIPLEX
-  "   -multiplex           enable the multiplexor VFS\n"
-#endif
-  "   -newline SEP         set output row separator. Default: '\\n'\n"
-  "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
-  "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
-  "   -quote               set output mode to 'quote'\n"
-  "   -separator SEP       set output column separator. Default: '|'\n"
-  "   -stats               print memory stats before each finalize\n"
-  "   -version             show SQLite version\n"
-  "   -vfs NAME            use NAME as the default VFS\n"
-#ifdef SQLITE_ENABLE_VFSTRACE
-  "   -vfstrace            enable tracing of all VFS calls\n"
-#endif
-;
-static void usage(int showDetail){
-  utf8_printf(stderr,
-      "Usage: %s [OPTIONS] FILENAME [SQL]\n"
-      "FILENAME is the name of an SQLite database. A new database is created\n"
-      "if the file does not previously exist.\n", Argv0);
-  if( showDetail ){
-    utf8_printf(stderr, "OPTIONS include:\n%s", zOptions);
-  }else{
-    raw_printf(stderr, "Use the -help option for additional information\n");
-  }
-  exit(1);
-}
-
-/*
-** Initialize the state information in data
-*/
-static void main_init(ShellState *data) {
-  memset(data, 0, sizeof(*data));
-  data->normalMode = data->cMode = data->mode = MODE_List;
-  data->autoExplain = 1;
-  memcpy(data->colSeparator,SEP_Column, 2);
-  memcpy(data->rowSeparator,SEP_Row, 2);
-  data->showHeader = 0;
-  data->shellFlgs = SHFLG_Lookaside;
-  sqlite3_config(SQLITE_CONFIG_URI, 1);
-  sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
-  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
-  sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
-  sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
-}
-
-/*
-** Output text to the console in a font that attracts extra attention.
-*/
-#ifdef _WIN32
-static void printBold(const char *zText){
-  HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE);
-  CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo;
-  GetConsoleScreenBufferInfo(out, &defaultScreenInfo);
-  SetConsoleTextAttribute(out,
-         FOREGROUND_RED|FOREGROUND_INTENSITY
-  );
-  printf("%s", zText);
-  SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
-}
-#else
-static void printBold(const char *zText){
-  printf("\033[1m%s\033[0m", zText);
-}
-#endif
-
-/*
-** Get the argument to an --option.  Throw an error and die if no argument
-** is available.
-*/
-static char *cmdline_option_value(int argc, char **argv, int i){
-  if( i==argc ){
-    utf8_printf(stderr, "%s: Error: missing argument to %s\n",
-            argv[0], argv[argc-1]);
-    exit(1);
-  }
-  return argv[i];
-}
-
-#ifndef SQLITE_SHELL_IS_UTF8
-#  if (defined(_WIN32) || defined(WIN32)) && defined(_MSC_VER)
-#    define SQLITE_SHELL_IS_UTF8          (0)
-#  else
-#    define SQLITE_SHELL_IS_UTF8          (1)
-#  endif
-#endif
-
-#if SQLITE_SHELL_IS_UTF8
-int SQLITE_CDECL main(int argc, char **argv){
-#else
-int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
-  char **argv;
-#endif
-  char *zErrMsg = 0;
-  ShellState data;
-  const char *zInitFile = 0;
-  int i;
-  int rc = 0;
-  int warnInmemoryDb = 0;
-  int readStdin = 1;
-  int nCmd = 0;
-  char **azCmd = 0;
-
-  setBinaryMode(stdin, 0);
-  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
-  stdin_is_interactive = isatty(0);
-  stdout_is_console = isatty(1);
-
-#if USE_SYSTEM_SQLITE+0!=1
-  if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
-    utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
-            sqlite3_sourceid(), SQLITE_SOURCE_ID);
-    exit(1);
-  }
-#endif
-  main_init(&data);
-#if !SQLITE_SHELL_IS_UTF8
-  sqlite3_initialize();
-  argv = sqlite3_malloc64(sizeof(argv[0])*argc);
-  if( argv==0 ){
-    raw_printf(stderr, "out of memory\n");
-    exit(1);
-  }
-  for(i=0; i<argc; i++){
-    argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]);
-    if( argv[i]==0 ){
-      raw_printf(stderr, "out of memory\n");
-      exit(1);
-    }
-  }
-#endif
-  assert( argc>=1 && argv && argv[0] );
-  Argv0 = argv[0];
-
-  /* Make sure we have a valid signal handler early, before anything
-  ** else is done.
-  */
-#ifdef SIGINT
-  signal(SIGINT, interrupt_handler);
-#endif
-
-#ifdef SQLITE_SHELL_DBNAME_PROC
-  {
-    /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name
-    ** of a C-function that will provide the name of the database file.  Use
-    ** this compile-time option to embed this shell program in larger
-    ** applications. */
-    extern void SQLITE_SHELL_DBNAME_PROC(const char**);
-    SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename);
-    warnInmemoryDb = 0;
-  }
-#endif
-
-  /* Do an initial pass through the command-line argument to locate
-  ** the name of the database file, the name of the initialization file,
-  ** the size of the alternative malloc heap,
-  ** and the first command to execute.
-  */
-  for(i=1; i<argc; i++){
-    char *z;
-    z = argv[i];
-    if( z[0]!='-' ){
-      if( data.zDbFilename==0 ){
-        data.zDbFilename = z;
-      }else{
-        /* Excesss arguments are interpreted as SQL (or dot-commands) and
-        ** mean that nothing is read from stdin */
-        readStdin = 0;
-        nCmd++;
-        azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd);
-        if( azCmd==0 ){
-          raw_printf(stderr, "out of memory\n");
-          exit(1);
-        }
-        azCmd[nCmd-1] = z;
-      }
-    }
-    if( z[1]=='-' ) z++;
-    if( strcmp(z,"-separator")==0
-     || strcmp(z,"-nullvalue")==0
-     || strcmp(z,"-newline")==0
-     || strcmp(z,"-cmd")==0
-    ){
-      (void)cmdline_option_value(argc, argv, ++i);
-    }else if( strcmp(z,"-init")==0 ){
-      zInitFile = cmdline_option_value(argc, argv, ++i);
-    }else if( strcmp(z,"-batch")==0 ){
-      /* Need to check for batch mode here to so we can avoid printing
-      ** informational messages (like from process_sqliterc) before
-      ** we do the actual processing of arguments later in a second pass.
-      */
-      stdin_is_interactive = 0;
-    }else if( strcmp(z,"-heap")==0 ){
-#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
-      const char *zSize;
-      sqlite3_int64 szHeap;
-
-      zSize = cmdline_option_value(argc, argv, ++i);
-      szHeap = integerValue(zSize);
-      if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000;
-      sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64);
-#else
-      (void)cmdline_option_value(argc, argv, ++i);
-#endif
-    }else if( strcmp(z,"-pagecache")==0 ){
-      int n, sz;
-      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( sz>70000 ) sz = 70000;
-      if( sz<0 ) sz = 0;
-      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      sqlite3_config(SQLITE_CONFIG_PAGECACHE,
-                    (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
-      data.shellFlgs |= SHFLG_Pagecache;
-    }else if( strcmp(z,"-lookaside")==0 ){
-      int n, sz;
-      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( sz<0 ) sz = 0;
-      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( n<0 ) n = 0;
-      sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n);
-      if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside;
-#ifdef SQLITE_ENABLE_VFSTRACE
-    }else if( strcmp(z,"-vfstrace")==0 ){
-      extern int vfstrace_register(
-         const char *zTraceName,
-         const char *zOldVfsName,
-         int (*xOut)(const char*,void*),
-         void *pOutArg,
-         int makeDefault
-      );
-      vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
-#endif
-#ifdef SQLITE_ENABLE_MULTIPLEX
-    }else if( strcmp(z,"-multiplex")==0 ){
-      extern int sqlite3_multiple_initialize(const char*,int);
-      sqlite3_multiplex_initialize(0, 1);
-#endif
-    }else if( strcmp(z,"-mmap")==0 ){
-      sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
-      sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
-    }else if( strcmp(z,"-vfs")==0 ){
-      sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i));
-      if( pVfs ){
-        sqlite3_vfs_register(pVfs, 1);
-      }else{
-        utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
-        exit(1);
-      }
-    }
-  }
-  if( data.zDbFilename==0 ){
-#ifndef SQLITE_OMIT_MEMORYDB
-    data.zDbFilename = ":memory:";
-    warnInmemoryDb = argc==1;
-#else
-    utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0);
-    return 1;
-#endif
-  }
-  data.out = stdout;
-
-  /* Go ahead and open the database file if it already exists.  If the
-  ** file does not exist, delay opening it.  This prevents empty database
-  ** files from being created if a user mistypes the database name argument
-  ** to the sqlite command-line tool.
-  */
-  if( access(data.zDbFilename, 0)==0 ){
-    open_db(&data, 0);
-  }
-
-  /* Process the initialization file if there is one.  If no -init option
-  ** is given on the command line, look for a file named ~/.sqliterc and
-  ** try to process it.
-  */
-  process_sqliterc(&data,zInitFile);
-
-  /* Make a second pass through the command-line argument and set
-  ** options.  This second pass is delayed until after the initialization
-  ** file is processed so that the command-line arguments will override
-  ** settings in the initialization file.
-  */
-  for(i=1; i<argc; i++){
-    char *z = argv[i];
-    if( z[0]!='-' ) continue;
-    if( z[1]=='-' ){ z++; }
-    if( strcmp(z,"-init")==0 ){
-      i++;
-    }else if( strcmp(z,"-html")==0 ){
-      data.mode = MODE_Html;
-    }else if( strcmp(z,"-list")==0 ){
-      data.mode = MODE_List;
-    }else if( strcmp(z,"-quote")==0 ){
-      data.mode = MODE_Quote;
-    }else if( strcmp(z,"-line")==0 ){
-      data.mode = MODE_Line;
-    }else if( strcmp(z,"-column")==0 ){
-      data.mode = MODE_Column;
-    }else if( strcmp(z,"-csv")==0 ){
-      data.mode = MODE_Csv;
-      memcpy(data.colSeparator,",",2);
-    }else if( strcmp(z,"-ascii")==0 ){
-      data.mode = MODE_Ascii;
-      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
-                       SEP_Unit);
-      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,
-                       SEP_Record);
-    }else if( strcmp(z,"-separator")==0 ){
-      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
-                       "%s",cmdline_option_value(argc,argv,++i));
-    }else if( strcmp(z,"-newline")==0 ){
-      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,
-                       "%s",cmdline_option_value(argc,argv,++i));
-    }else if( strcmp(z,"-nullvalue")==0 ){
-      sqlite3_snprintf(sizeof(data.nullValue), data.nullValue,
-                       "%s",cmdline_option_value(argc,argv,++i));
-    }else if( strcmp(z,"-header")==0 ){
-      data.showHeader = 1;
-    }else if( strcmp(z,"-noheader")==0 ){
-      data.showHeader = 0;
-    }else if( strcmp(z,"-echo")==0 ){
-      ShellSetFlag(&data, SHFLG_Echo);
-    }else if( strcmp(z,"-eqp")==0 ){
-      data.autoEQP = 1;
-    }else if( strcmp(z,"-eqpfull")==0 ){
-      data.autoEQP = 2;
-    }else if( strcmp(z,"-stats")==0 ){
-      data.statsOn = 1;
-    }else if( strcmp(z,"-scanstats")==0 ){
-      data.scanstatsOn = 1;
-    }else if( strcmp(z,"-backslash")==0 ){
-      /* Undocumented command-line option: -backslash
-      ** Causes C-style backslash escapes to be evaluated in SQL statements
-      ** prior to sending the SQL into SQLite.  Useful for injecting
-      ** crazy bytes in the middle of SQL statements for testing and debugging.
-      */
-      ShellSetFlag(&data, SHFLG_Backslash);
-    }else if( strcmp(z,"-bail")==0 ){
-      bail_on_error = 1;
-    }else if( strcmp(z,"-version")==0 ){
-      printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid());
-      return 0;
-    }else if( strcmp(z,"-interactive")==0 ){
-      stdin_is_interactive = 1;
-    }else if( strcmp(z,"-batch")==0 ){
-      stdin_is_interactive = 0;
-    }else if( strcmp(z,"-heap")==0 ){
-      i++;
-    }else if( strcmp(z,"-pagecache")==0 ){
-      i+=2;
-    }else if( strcmp(z,"-lookaside")==0 ){
-      i+=2;
-    }else if( strcmp(z,"-mmap")==0 ){
-      i++;
-    }else if( strcmp(z,"-vfs")==0 ){
-      i++;
-#ifdef SQLITE_ENABLE_VFSTRACE
-    }else if( strcmp(z,"-vfstrace")==0 ){
-      i++;
-#endif
-#ifdef SQLITE_ENABLE_MULTIPLEX
-    }else if( strcmp(z,"-multiplex")==0 ){
-      i++;
-#endif
-    }else if( strcmp(z,"-help")==0 ){
-      usage(1);
-    }else if( strcmp(z,"-cmd")==0 ){
-      /* Run commands that follow -cmd first and separately from commands
-      ** that simply appear on the command-line.  This seems goofy.  It would
-      ** be better if all commands ran in the order that they appear.  But
-      ** we retain the goofy behavior for historical compatibility. */
-      if( i==argc-1 ) break;
-      z = cmdline_option_value(argc,argv,++i);
-      if( z[0]=='.' ){
-        rc = do_meta_command(z, &data);
-        if( rc && bail_on_error ) return rc==2 ? 0 : rc;
-      }else{
-        open_db(&data, 0);
-        rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg);
-        if( zErrMsg!=0 ){
-          utf8_printf(stderr,"Error: %s\n", zErrMsg);
-          if( bail_on_error ) return rc!=0 ? rc : 1;
-        }else if( rc!=0 ){
-          utf8_printf(stderr,"Error: unable to process SQL \"%s\"\n", z);
-          if( bail_on_error ) return rc;
-        }
-      }
-    }else{
-      utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
-      raw_printf(stderr,"Use -help for a list of options.\n");
-      return 1;
-    }
-    data.cMode = data.mode;
-  }
-
-  if( !readStdin ){
-    /* Run all arguments that do not begin with '-' as if they were separate
-    ** command-line inputs, except for the argToSkip argument which contains
-    ** the database filename.
-    */
-    for(i=0; i<nCmd; i++){
-      if( azCmd[i][0]=='.' ){
-        rc = do_meta_command(azCmd[i], &data);
-        if( rc ) return rc==2 ? 0 : rc;
-      }else{
-        open_db(&data, 0);
-        rc = shell_exec(data.db, azCmd[i], shell_callback, &data, &zErrMsg);
-        if( zErrMsg!=0 ){
-          utf8_printf(stderr,"Error: %s\n", zErrMsg);
-          return rc!=0 ? rc : 1;
-        }else if( rc!=0 ){
-          utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]);
-          return rc;
-        }
-      }
-    }
-    free(azCmd);
-  }else{
-    /* Run commands received from standard input
-    */
-    if( stdin_is_interactive ){
-      char *zHome;
-      char *zHistory = 0;
-      int nHistory;
-      printf(
-        "SQLite version %s %.19s\n" /*extra-version-info*/
-        "Enter \".help\" for usage hints.\n",
-        sqlite3_libversion(), sqlite3_sourceid()
-      );
-      if( warnInmemoryDb ){
-        printf("Connected to a ");
-        printBold("transient in-memory database");
-        printf(".\nUse \".open FILENAME\" to reopen on a "
-               "persistent database.\n");
-      }
-      zHome = find_home_dir(0);
-      if( zHome ){
-        nHistory = strlen30(zHome) + 20;
-        if( (zHistory = malloc(nHistory))!=0 ){
-          sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome);
-        }
-      }
-      if( zHistory ){ shell_read_history(zHistory); }
-#if HAVE_READLINE || HAVE_EDITLINE
-      rl_attempted_completion_function = readline_completion;
-#elif HAVE_LINENOISE
-      linenoiseSetCompletionCallback(linenoise_completion);
-#endif
-      rc = process_input(&data, 0);
-      if( zHistory ){
-        shell_stifle_history(2000);
-        shell_write_history(zHistory);
-        free(zHistory);
-      }
-    }else{
-      rc = process_input(&data, stdin);
-    }
-  }
-  set_table_name(&data, 0);
-  if( data.db ){
-    session_close_all(&data);
-    sqlite3_close(data.db);
-  }
-  sqlite3_free(data.zFreeOnClose);
-  find_home_dir(1);
-#if !SQLITE_SHELL_IS_UTF8
-  for(i=0; i<argc; i++) sqlite3_free(argv[i]);
-  sqlite3_free(argv);
-#endif
-  return rc;
-}
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index 7ce9dc4431..807a169826 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -1,11 +1,16 @@
 #!/usr/bin/tclsh
 #
-# Run this script to generate the "src/shell.c" source file from 
+# Run this script to generate the "shell.c" source file from 
 # constituent parts.
 #
+# No arguments are required.  This script determines the location
+# of its input files relative to the location of the script itself.
+# This script should be tool/mkshellc.tcl.  If the directory holding
+# the script is $DIR, then the component parts are located in $DIR/../src
+# and $DIR/../ext/misc.
+#
 set topdir [file dir [file dir [file normal $argv0]]]
-puts "Overwriting $topdir/src/shell.c with new shell source code..."
-set out [open $topdir/src/shell.c wb]
+set out stdout
 puts $out {/* DO NOT EDIT!
 ** This file is automatically generated by the script in the canonical
 ** SQLite source tree at tool/mkshellc.tcl.  That script combines source

From 0a386468065688eadf2d3e8fcfc22b3c3bad90ba Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 14:13:20 +0000
Subject: [PATCH 182/270] Fix the "snapshot-tarball" makefile target, which was
 broken by the shell.c change of check-in [36acc0a9].

FossilOrigin-Name: c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339
---
 manifest               | 14 +++++++-------
 manifest.uuid          |  2 +-
 tool/mkautoconfamal.sh |  2 +-
 tool/warnings-clang.sh |  6 +++---
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index f63bc613e1..2e528e87e0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sfixes\sfrom\strunk.\s\sAll\schanges\sare\son\smakefiles\sand\stest\sscripts.\nThere\sare\sno\score\scode\schanges.
-D 2017-10-12T14:03:09.021
+C Fix\sthe\s"snapshot-tarball"\smakefile\starget,\swhich\swas\sbroken\sby\sthe\sshell.c\nchange\sof\scheck-in\s[36acc0a9].
+D 2017-10-12T14:13:20.758
 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20
@@ -1585,7 +1585,7 @@ F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
-F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7
+F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c736fdeca5e
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c
 F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4
@@ -1632,7 +1632,7 @@ F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4
 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003
 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c
 F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
-F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
+F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 8a4acea31e0f9c562949a2d767329533c0930d699ea19c6704c0ca0aa9154068
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5d0ceb8dcdef92cd72307e532a4a6c269b2c458fecb0bbede0bb941099eebc5b 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956
-R 55fc95c237c6d381b00d7c404f68ae59
+P 1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad
+R 8088fa2748c2b3c07e15e47d6681488a
 U drh
-Z 9ae511da21bc6dca261bd7a352a2c613
+Z 449b637c9672f868a961d2fed56fa177
diff --git a/manifest.uuid b/manifest.uuid
index 9e90e52ce3..0ade2a082b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad
\ No newline at end of file
+c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339
\ No newline at end of file
diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh
index 4864ee85c3..7cd7da35f6 100644
--- a/tool/mkautoconfamal.sh
+++ b/tool/mkautoconfamal.sh
@@ -51,7 +51,7 @@ cp sqlite3.h              $TMPSPACE
 cp sqlite3ext.h           $TMPSPACE
 cp $TOP/sqlite3.1         $TMPSPACE
 cp $TOP/sqlite3.pc.in     $TMPSPACE
-cp $TOP/src/shell.c       $TMPSPACE
+cp shell.c                $TMPSPACE
 cp $TOP/src/sqlite3.rc    $TMPSPACE
 cp $TOP/tool/Replace.cs   $TMPSPACE
 
diff --git a/tool/warnings-clang.sh b/tool/warnings-clang.sh
index 7a0aa4bce7..6dcc086d2f 100644
--- a/tool/warnings-clang.sh
+++ b/tool/warnings-clang.sh
@@ -3,12 +3,12 @@
 # Run this script in a directory with a working makefile to check for 
 # compiler warnings in SQLite.
 #
-rm -f sqlite3.c
-make sqlite3.c
+rm -f sqlite3.c shell.c
+make sqlite3.c shell.c
 echo '************* FTS4 and RTREE ****************'
 scan-build gcc -c -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
       -DSQLITE_DEBUG -DSQLITE_ENABLE_STAT3 sqlite3.c 2>&1 | grep -v 'ANALYZE:'
 echo '********** ENABLE_STAT3. THREADSAFE=0 *******'
 scan-build gcc -c -I. -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \
       -DSQLITE_DEBUG \
-      sqlite3.c ../sqlite/src/shell.c -ldl 2>&1 | grep -v 'ANALYZE:'
+      sqlite3.c shell.c -ldl 2>&1 | grep -v 'ANALYZE:'

From 6235ee5784bba91d58cf485176c0b5ad4177080e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 14:18:38 +0000
Subject: [PATCH 183/270] Fix a harmless compiler warning about an unused
 parameter.

FossilOrigin-Name: de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 2e528e87e0..b720d2abb5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s"snapshot-tarball"\smakefile\starget,\swhich\swas\sbroken\sby\sthe\sshell.c\nchange\sof\scheck-in\s[36acc0a9].
-D 2017-10-12T14:13:20.758
+C Fix\sa\sharmless\scompiler\swarning\sabout\san\sunused\sparameter.
+D 2017-10-12T14:18:38.182
 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20
@@ -445,7 +445,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 62f88892d3a2c68cff6e8f96c81c5dfe5178eace887880c36364aabe4d8d6422
+F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
@@ -1656,7 +1656,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1fb87a0c6e4db1f8bf51d552d9af1252544b7d7bd2dd80f78870f3b4fd347bad
-R 8088fa2748c2b3c07e15e47d6681488a
+P c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339
+R 6b8c8a980bca85216c2bbb28216f8b36
 U drh
-Z 449b637c9672f868a961d2fed56fa177
+Z ee2d81a451eaa5a0334a9c3d00838821
diff --git a/manifest.uuid b/manifest.uuid
index 0ade2a082b..0b314a09be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339
\ No newline at end of file
+de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 2ddca9a5f7..dbb7636ca7 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -1208,6 +1208,8 @@ static int jrnlBufferSize(Pager *pPager){
 
   assert( isOpen(pPager->fd) );
   dc = sqlite3OsDeviceCharacteristics(pPager->fd);
+#else
+  UNUSED_PARAMETER(pPager);
 #endif
 
 #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE

From 277ee81babd94ce08cda6c57596759dec457f35b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 19:50:28 +0000
Subject: [PATCH 184/270] Create the new ext/repair folder and move
 checkfreelist.c there.  Remove checkfreelist.c from the command-line shell
 (undoing check-in [48418f2e]).

FossilOrigin-Name: dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab
---
 ext/repair/README.md                 |  16 ++
 ext/{misc => repair}/checkfreelist.c |   0
 manifest                             |  18 +-
 manifest.uuid                        |   2 +-
 src/shell.c                          | 302 ---------------------------
 src/shell.c.in                       |   2 -
 6 files changed, 27 insertions(+), 313 deletions(-)
 create mode 100644 ext/repair/README.md
 rename ext/{misc => repair}/checkfreelist.c (100%)

diff --git a/ext/repair/README.md b/ext/repair/README.md
new file mode 100644
index 0000000000..927ceb7c44
--- /dev/null
+++ b/ext/repair/README.md
@@ -0,0 +1,16 @@
+This folder contains extensions and utility programs intended to analyze
+live database files, detect problems, and possibly fix them.
+
+As SQLite is being used on larger and larger databases, database sizes
+are growing into the terabyte range.  At that size, hardware malfunctions
+and/or cosmic rays will occasionally corrupt a database file.  Detecting 
+problems and fixing errors a terabyte-sized databases can take hours or days,
+and it is undesirable to take applications that depend on the databases 
+off-line for such a long time.
+The utilities in the folder are intended to provide mechanisms for
+detecting and fixing problems in large databases while those databases
+are in active use.
+
+The utilities and extensions in this folder are experimental and under
+active development at the time of this writing (2017-10-12).  If and when
+they stabilize, this README will be updated to reflect that fact.
diff --git a/ext/misc/checkfreelist.c b/ext/repair/checkfreelist.c
similarity index 100%
rename from ext/misc/checkfreelist.c
rename to ext/repair/checkfreelist.c
diff --git a/manifest b/manifest
index e5589b7672..5c8309403b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sfixes\sfrom\sthe\s3.21\sbranch.
-D 2017-10-12T01:24:36.582
+C Create\sthe\snew\sext/repair\sfolder\sand\smove\scheckfreelist.c\sthere.\s\sRemove\ncheckfreelist.c\sfrom\sthe\scommand-line\sshell\s(undoing\scheck-in\s[48418f2e]).
+D 2017-10-12T19:50:28.969
 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 523fd246fd801689c766945f6450615d3784fda2dc04993bcafc796ac6afd8df
@@ -259,7 +259,6 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
-F ext/misc/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
@@ -326,6 +325,8 @@ F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa
 F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078da04feb
 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
+F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
+F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 w ext/misc/checkfreelist.c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -461,8 +462,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c ffb06532d6667bf1bb64080a316120c67249636a12f008c2f9716d6778565d57
-F src/shell.c.in 7842db264d5512520c61d0353196eeefeb65b710dd0d97d4ad9844c56e313be5
+F src/shell.c b1c14539ae8f756a96a5604952e24fb8f2a65745290037f4f43dddfabac76e6e
+F src/shell.c.in 73d8000bb066cd7ceb9655ffdb0e19a80779e3c64506f5a1ecfa9838511bee18
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1660,7 +1661,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52 f0a2724f0a255cd5a262f31e4ee1f99ae713c25a9ecc56dc794c95f223453b9b
-R 9b46eb5073fd5db6110bab2092230089
+P 18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857
+Q -48418f2ed5ab1cb270776166141ce32ed3ebf22ed4e33a66a204d4fde9d11f52
+R 4a0d06893cd053abce709ab7740cd918
 U drh
-Z b5c9a13e106677edbcab94b5f19597c5
+Z 46634faecfbbf65605db6fcff2dbe040
diff --git a/manifest.uuid b/manifest.uuid
index f8e3904fb9..58e6b4a3f7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-18d4654fd161900f98ff435ea9e0a3c44b9972f84ee9f43096f9998f844ff857
\ No newline at end of file
+dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index 6c14db9fdd..2b77d482e6 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -2156,307 +2156,6 @@ int sqlite3_completion_init(
 }
 
 /************************* End ../ext/misc/completion.c ********************/
-/************************* Begin ../ext/misc/checkfreelist.c ******************/
-/*
-** 2017 October 11
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This module exports a single C function:
-**
-**   int sqlite3_check_freelist(sqlite3 *db, const char *zDb);
-**
-** This function checks the free-list in database zDb (one of "main", 
-** "temp", etc.) and reports any errors by invoking the sqlite3_log()
-** function. It returns SQLITE_OK if successful, or an SQLite error
-** code otherwise. It is not an error if the free-list is corrupted but
-** no IO or OOM errors occur.
-**
-** If this file is compiled and loaded as an SQLite loadable extension,
-** it adds an SQL function "checkfreelist" to the database handle, to
-** be invoked as follows:
-**
-**   SELECT checkfreelist(<database-name>);
-**
-** This function performs the same checks as sqlite3_check_freelist(),
-** except that it returns all error messages as a single text value,
-** separated by newline characters. If the freelist is not corrupted
-** in any way, an empty string is returned.
-**
-** To compile this module for use as an SQLite loadable extension:
-**
-**   gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so
-*/
-
-SQLITE_EXTENSION_INIT1
-
-#ifndef SQLITE_AMALGAMATION
-# include <string.h>
-# include <stdio.h>
-# include <stdlib.h>
-# include <assert.h>
-# define ALWAYS(X)  1
-# define NEVER(X)   0
-  typedef unsigned char u8;
-  typedef unsigned short u16;
-  typedef unsigned int u32;
-#define get4byte(x) (        \
-    ((u32)((x)[0])<<24) +    \
-    ((u32)((x)[1])<<16) +    \
-    ((u32)((x)[2])<<8) +     \
-    ((u32)((x)[3]))          \
-)
-#endif
-
-/*
-** Execute a single PRAGMA statement and return the integer value returned
-** via output parameter (*pnOut).
-**
-** The SQL statement passed as the third argument should be a printf-style
-** format string containing a single "%s" which will be replace by the
-** value passed as the second argument. e.g.
-**
-**   sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut)
-**
-** executes "PRAGMA main.page_count" and stores the results in (*pnOut).
-*/
-static int sqlGetInteger(
-  sqlite3 *db,                    /* Database handle */
-  const char *zDb,                /* Database name ("main", "temp" etc.) */
-  const char *zFmt,               /* SQL statement format */
-  u32 *pnOut                      /* OUT: Integer value */
-){
-  int rc, rc2;
-  char *zSql;
-  sqlite3_stmt *pStmt = 0;
-  int bOk = 0;
-
-  zSql = sqlite3_mprintf(zFmt, zDb);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-    sqlite3_free(zSql);
-  }
-
-  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
-    *pnOut = (u32)sqlite3_column_int(pStmt, 0);
-    bOk = 1;
-  }
-
-  rc2 = sqlite3_finalize(pStmt);
-  if( rc==SQLITE_OK ) rc = rc2;
-  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR;
-  return rc;
-}
-
-/*
-** Argument zFmt must be a printf-style format string and must be 
-** followed by its required arguments. If argument pzOut is NULL, 
-** then the results of printf()ing the format string are passed to
-** sqlite3_log(). Otherwise, they are appended to the string
-** at (*pzOut).
-*/
-static int checkFreelistError(char **pzOut, const char *zFmt, ...){
-  int rc = SQLITE_OK;
-  char *zErr = 0;
-  va_list ap;
-
-  va_start(ap, zFmt);
-  zErr = sqlite3_vmprintf(zFmt, ap);
-  if( zErr==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    if( pzOut ){
-      *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr);
-      if( *pzOut==0 ) rc = SQLITE_NOMEM;
-    }else{
-      sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr);
-    }
-    sqlite3_free(zErr);
-  }
-  va_end(ap);
-  return rc;
-}
-
-static int checkFreelist(
-  sqlite3 *db, 
-  const char *zDb,
-  char **pzOut
-){
-  /* This query returns one row for each page on the free list. Each row has
-  ** two columns - the page number and page content.  */
-  const char *zTrunk = 
-    "WITH freelist_trunk(i, d, n) AS ("
-      "SELECT 1, NULL, sqlite_readint32(data, 32) "
-      "FROM sqlite_dbpage(:1) WHERE pgno=1 "
-        "UNION ALL "
-      "SELECT n, data, sqlite_readint32(data) "
-      "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n "
-    ")"
-    "SELECT i, d FROM freelist_trunk WHERE i!=1;";
-
-  int rc, rc2;                    /* Return code */
-  sqlite3_stmt *pTrunk = 0;       /* Compilation of zTrunk */
-  u32 nPage = 0;                  /* Number of pages in db */
-  u32 nExpected = 0;              /* Expected number of free pages */
-  u32 nFree = 0;                  /* Number of pages on free list */
-
-  if( zDb==0 ) zDb = "main";
-
-  if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage))
-   || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected))
-  ){
-    return rc;
-  }
-
-  rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0);
-  if( rc!=SQLITE_OK ) return rc;
-  sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC);
-  while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){
-    u32 i;
-    u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0);
-    const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1);
-    int nData = sqlite3_column_bytes(pTrunk, 1);
-    u32 iNext = get4byte(&aData[0]);
-    u32 nLeaf = get4byte(&aData[4]);
-
-    if( nLeaf>((nData/4)-2-6) ){
-      rc = checkFreelistError(pzOut, 
-          "leaf count out of range (%d) on trunk page %d", 
-          (int)nLeaf, (int)iTrunk
-      );
-      nLeaf = (nData/4) - 2 - 6;
-    }
-
-    nFree += 1+nLeaf;
-    if( iNext>nPage ){
-      rc = checkFreelistError(pzOut, 
-          "trunk page %d is out of range", (int)iNext
-      );
-    }
-
-    for(i=0; rc==SQLITE_OK && i<nLeaf; i++){
-      u32 iLeaf = get4byte(&aData[8 + 4*i]);
-      if( iLeaf==0 || iLeaf>nPage ){
-        rc = checkFreelistError(pzOut,
-            "leaf page %d is out of range (child %d of trunk page %d)", 
-            (int)iLeaf, (int)i, (int)iTrunk
-        );
-      }
-    }
-  }
-
-  if( rc==SQLITE_OK && nFree!=nExpected ){
-    rc = checkFreelistError(pzOut,
-        "free-list count mismatch: actual=%d header=%d", 
-        (int)nFree, (int)nExpected
-    );
-  }
-
-  rc2 = sqlite3_finalize(pTrunk);
-  if( rc==SQLITE_OK ) rc = rc2;
-  return rc;
-}
-
-int sqlite3_check_freelist(sqlite3 *db, const char *zDb){
-  return checkFreelist(db, zDb, 0);
-}
-
-static void checkfreelist_function(
-  sqlite3_context *pCtx,
-  int nArg,
-  sqlite3_value **apArg
-){
-  const char *zDb;
-  int rc;
-  char *zOut = 0;
-  sqlite3 *db = sqlite3_context_db_handle(pCtx);
-
-  assert( nArg==1 );
-  zDb = (const char*)sqlite3_value_text(apArg[0]);
-  rc = checkFreelist(db, zDb, &zOut);
-  if( rc==SQLITE_OK ){
-    sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT);
-  }else{
-    sqlite3_result_error_code(pCtx, rc);
-  }
-
-  sqlite3_free(zOut);
-}
-
-/*
-** An SQL function invoked as follows:
-**
-**   sqlite_readint32(BLOB)           -- Decode 32-bit integer from start of blob
-*/
-static void readint_function(
-  sqlite3_context *pCtx,
-  int nArg,
-  sqlite3_value **apArg
-){
-  const u8 *zBlob;
-  int nBlob;
-  int iOff = 0;
-  u32 iRet = 0;
-
-  if( nArg!=1 && nArg!=2 ){
-    sqlite3_result_error(
-        pCtx, "wrong number of arguments to function sqlite_readint32()", -1
-    );
-    return;
-  }
-  if( nArg==2 ){
-    iOff = sqlite3_value_int(apArg[1]);
-  }
-
-  zBlob = sqlite3_value_blob(apArg[0]);
-  nBlob = sqlite3_value_bytes(apArg[0]);
-
-  if( nBlob>=(iOff+4) ){
-    iRet = get4byte(&zBlob[iOff]);
-  }
-
-  sqlite3_result_int64(pCtx, (sqlite3_int64)iRet);
-}
-
-/*
-** Register the SQL functions.
-*/
-static int cflRegister(sqlite3 *db){
-  int rc = sqlite3_create_function(
-      db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0
-  );
-  if( rc!=SQLITE_OK ) return rc;
-  rc = sqlite3_create_function(
-      db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0
-  );
-  return rc;
-}
-
-/*
-** Extension load function.
-*/
-#ifdef _WIN32
-
-#endif
-int sqlite3_checkfreelist_init(
-  sqlite3 *db, 
-  char **pzErrMsg, 
-  const sqlite3_api_routines *pApi
-){
-  SQLITE_EXTENSION_INIT2(pApi);
-  return cflRegister(db);
-}
-
-/************************* End ../ext/misc/checkfreelist.c ********************/
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -4547,7 +4246,6 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
-    sqlite3_checkfreelist_init(p->db, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }
diff --git a/src/shell.c.in b/src/shell.c.in
index de170887f7..54a61b9456 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -796,7 +796,6 @@ static void shellAddSchemaName(
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
-INCLUDE ../ext/misc/checkfreelist.c
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -2887,7 +2886,6 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
-    sqlite3_checkfreelist_init(p->db, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }

From c318f730a44618db5a84bba635be5b53ab876dcc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 13 Oct 2017 15:06:06 +0000
Subject: [PATCH 185/270] Move some test logic out of tclsqlite.c and into
 auxiliary test_*.c files. This is a work in progress.

FossilOrigin-Name: 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf
---
 Makefile.in      |   2 +
 Makefile.msc     |   2 +
 main.mk          |   2 +
 manifest         |  24 +-
 manifest.uuid    |   2 +-
 src/tclsqlite.c  | 636 +++--------------------------------------------
 src/test_md5.c   | 450 +++++++++++++++++++++++++++++++++
 src/test_tclsh.c | 196 +++++++++++++++
 8 files changed, 697 insertions(+), 617 deletions(-)
 create mode 100644 src/test_md5.c
 create mode 100644 src/test_tclsh.c

diff --git a/Makefile.in b/Makefile.in
index 5295c2b032..8c306629e0 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -396,6 +396,7 @@ TESTSRC = \
   $(TOP)/src/test_intarray.c \
   $(TOP)/src/test_journal.c \
   $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
   $(TOP)/src/test_multiplex.c \
   $(TOP)/src/test_mutex.c \
   $(TOP)/src/test_onefile.c \
@@ -407,6 +408,7 @@ TESTSRC = \
   $(TOP)/src/test_server.c \
   $(TOP)/src/test_superlock.c \
   $(TOP)/src/test_syscall.c \
+  $(TOP)/src/test_tclsh.c \
   $(TOP)/src/test_tclvar.c \
   $(TOP)/src/test_thread.c \
   $(TOP)/src/test_vfs.c \
diff --git a/Makefile.msc b/Makefile.msc
index da71df89f0..e4a72e872b 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1374,6 +1374,7 @@ TESTSRC = \
   $(TOP)\src\test_intarray.c \
   $(TOP)\src\test_journal.c \
   $(TOP)\src\test_malloc.c \
+  $(TOP)\src\test_md5.c \
   $(TOP)\src\test_multiplex.c \
   $(TOP)\src\test_mutex.c \
   $(TOP)\src\test_onefile.c \
@@ -1385,6 +1386,7 @@ TESTSRC = \
   $(TOP)\src\test_server.c \
   $(TOP)\src\test_superlock.c \
   $(TOP)\src\test_syscall.c \
+  $(TOP)\src\test_tclsh.c \
   $(TOP)\src\test_tclvar.c \
   $(TOP)\src\test_thread.c \
   $(TOP)\src\test_vfs.c \
diff --git a/main.mk b/main.mk
index d03ee8d1d3..84d1a3eb76 100644
--- a/main.mk
+++ b/main.mk
@@ -308,6 +308,7 @@ TESTSRC = \
   $(TOP)/src/test_intarray.c \
   $(TOP)/src/test_journal.c \
   $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
   $(TOP)/src/test_multiplex.c \
   $(TOP)/src/test_mutex.c \
   $(TOP)/src/test_onefile.c \
@@ -320,6 +321,7 @@ TESTSRC = \
   $(TOP)/src/test_sqllog.c \
   $(TOP)/src/test_superlock.c \
   $(TOP)/src/test_syscall.c \
+  $(TOP)/src/test_tclsh.c \
   $(TOP)/src/test_tclvar.c \
   $(TOP)/src/test_thread.c \
   $(TOP)/src/test_vfs.c \
diff --git a/manifest b/manifest
index 3360e30a3a..250e0c8001 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Add\sthe\ssqlite_dbpage\svirtual\stable\s(enabled\susing\sSQLITE_ENABLE_DBPAGE_VTAB).\nMake\sthat\svirtual\stable\sand\sdbstat\savailable\sto\sthe\scommand-line\sshell.
-D 2017-10-12T20:37:20.278
-F Makefile.in cb88ca5a6d8e116a50bd19bf477384498df475c5463cbbf53f36624ce308ed62
+C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress.
+D 2017-10-13T15:06:06.783
+F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 918f07fee01bf858d20df6b37802161423f216da1ece14794afee12de4992e35
+F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 8be17ffd37df86bfbd696072045cd360600499876f7144313090f96ce3a37a55
+F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828
+F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2
 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
@@ -499,6 +499,7 @@ F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c
 F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3
 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd
 F src/test_malloc.c 4f06a805de86be5216a127b3777ca2d5a1ff99d1a9238374ce136a47411be36c
+F src/test_md5.c 7268e1e8c399d4a5e181b64ac20e1e6f3bc4dd9fc87abac02db145a3d951fa8c
 F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e
 F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635
 F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a
@@ -513,6 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
 F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef
 F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
+F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
@@ -1661,8 +1663,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956 dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab
-R a8e3c658e72c296e51cc10d88be952b0
-T +closed dfdebd12bfc80b91d234ab328cb6106d5d37ccb79b58e36e556c1a8af640a4ab
+P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a
+R 225e4cd689e0c61d406db3daa52debdc
+T *branch * tclsqlite-cleanup
+T *sym-tclsqlite-cleanup *
+T -sym-trunk *
 U drh
-Z 03ff5b730193797e00df33319c2ae82d
+Z 604c544fbf5b9e259c249448beb617c7
diff --git a/manifest.uuid b/manifest.uuid
index 50f9e1660f..705f05a85e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a
\ No newline at end of file
+95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 1b9f91405e..a66fe6832e 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -3546,443 +3546,16 @@ int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
 int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
 #endif
 
+/*
+** If the TCLSH macro is defined to be either 1 or 2, then a main()
+** routine is inserted that starts up a Tcl interpreter.  When TCLSH==1
+** the interpreter works like an ordinary tclsh.  When TCLSH==2 then the
+** startup script is supplied by an routine named "tclsh_main_loop()"
+** that must be linked separately.  The TCLSH==2 technique is used to
+** generate stand-alone executables based on TCL, such as 
+** sqlite3_analyzer.exe.
+*/
 #ifdef TCLSH
-/*****************************************************************************
-** All of the code that follows is used to build standalone TCL interpreters
-** that are statically linked with SQLite.  Enable these by compiling
-** with -DTCLSH=n where n can be 1 or 2.  An n of 1 generates a standard
-** tclsh but with SQLite built in.  An n of 2 generates the SQLite space
-** analysis program.
-*/
-
-#if defined(SQLITE_TEST) || defined(SQLITE_TCLMD5)
-/*
- * This code implements the MD5 message-digest algorithm.
- * The algorithm is due to Ron Rivest.  This code was
- * written by Colin Plumb in 1993, no copyright is claimed.
- * This code is in the public domain; do with it what you wish.
- *
- * Equivalent code is available from RSA Data Security, Inc.
- * This code has been tested against that, and is equivalent,
- * except that you don't need to include two pages of legalese
- * with every copy.
- *
- * To compute the message digest of a chunk of bytes, declare an
- * MD5Context structure, pass it to MD5Init, call MD5Update as
- * needed on buffers full of bytes, and then call MD5Final, which
- * will fill a supplied 16-byte array with the digest.
- */
-
-/*
- * If compiled on a machine that doesn't have a 32-bit integer,
- * you just set "uint32" to the appropriate datatype for an
- * unsigned 32-bit integer.  For example:
- *
- *       cc -Duint32='unsigned long' md5.c
- *
- */
-#ifndef uint32
-#  define uint32 unsigned int
-#endif
-
-struct MD5Context {
-  int isInit;
-  uint32 buf[4];
-  uint32 bits[2];
-  unsigned char in[64];
-};
-typedef struct MD5Context MD5Context;
-
-/*
- * Note: this code is harmless on little-endian machines.
- */
-static void byteReverse (unsigned char *buf, unsigned longs){
-        uint32 t;
-        do {
-                t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 |
-                            ((unsigned)buf[1]<<8 | buf[0]);
-                *(uint32 *)buf = t;
-                buf += 4;
-        } while (--longs);
-}
-/* The four core functions - F1 is optimized somewhat */
-
-/* #define F1(x, y, z) (x & y | ~x & z) */
-#define F1(x, y, z) (z ^ (x & (y ^ z)))
-#define F2(x, y, z) F1(z, x, y)
-#define F3(x, y, z) (x ^ y ^ z)
-#define F4(x, y, z) (y ^ (x | ~z))
-
-/* This is the central step in the MD5 algorithm. */
-#define MD5STEP(f, w, x, y, z, data, s) \
-        ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
-
-/*
- * The core of the MD5 algorithm, this alters an existing MD5 hash to
- * reflect the addition of 16 longwords of new data.  MD5Update blocks
- * the data and converts bytes into longwords for this routine.
- */
-static void MD5Transform(uint32 buf[4], const uint32 in[16]){
-        register uint32 a, b, c, d;
-
-        a = buf[0];
-        b = buf[1];
-        c = buf[2];
-        d = buf[3];
-
-        MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478,  7);
-        MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
-        MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
-        MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
-        MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf,  7);
-        MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
-        MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
-        MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
-        MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8,  7);
-        MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
-        MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
-        MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
-        MD5STEP(F1, a, b, c, d, in[12]+0x6b901122,  7);
-        MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
-        MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
-        MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
-
-        MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562,  5);
-        MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340,  9);
-        MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
-        MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
-        MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d,  5);
-        MD5STEP(F2, d, a, b, c, in[10]+0x02441453,  9);
-        MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
-        MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
-        MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6,  5);
-        MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6,  9);
-        MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
-        MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
-        MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905,  5);
-        MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8,  9);
-        MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
-        MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
-
-        MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942,  4);
-        MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
-        MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
-        MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
-        MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44,  4);
-        MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
-        MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
-        MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
-        MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6,  4);
-        MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
-        MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
-        MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
-        MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039,  4);
-        MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
-        MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
-        MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
-
-        MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244,  6);
-        MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
-        MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
-        MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
-        MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3,  6);
-        MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
-        MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
-        MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
-        MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f,  6);
-        MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
-        MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
-        MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
-        MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82,  6);
-        MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
-        MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
-        MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
-
-        buf[0] += a;
-        buf[1] += b;
-        buf[2] += c;
-        buf[3] += d;
-}
-
-/*
- * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
- * initialization constants.
- */
-static void MD5Init(MD5Context *ctx){
-        ctx->isInit = 1;
-        ctx->buf[0] = 0x67452301;
-        ctx->buf[1] = 0xefcdab89;
-        ctx->buf[2] = 0x98badcfe;
-        ctx->buf[3] = 0x10325476;
-        ctx->bits[0] = 0;
-        ctx->bits[1] = 0;
-}
-
-/*
- * Update context to reflect the concatenation of another buffer full
- * of bytes.
- */
-static
-void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
-        uint32 t;
-
-        /* Update bitcount */
-
-        t = ctx->bits[0];
-        if ((ctx->bits[0] = t + ((uint32)len << 3)) < t)
-                ctx->bits[1]++; /* Carry from low to high */
-        ctx->bits[1] += len >> 29;
-
-        t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
-
-        /* Handle any leading odd-sized chunks */
-
-        if ( t ) {
-                unsigned char *p = (unsigned char *)ctx->in + t;
-
-                t = 64-t;
-                if (len < t) {
-                        memcpy(p, buf, len);
-                        return;
-                }
-                memcpy(p, buf, t);
-                byteReverse(ctx->in, 16);
-                MD5Transform(ctx->buf, (uint32 *)ctx->in);
-                buf += t;
-                len -= t;
-        }
-
-        /* Process data in 64-byte chunks */
-
-        while (len >= 64) {
-                memcpy(ctx->in, buf, 64);
-                byteReverse(ctx->in, 16);
-                MD5Transform(ctx->buf, (uint32 *)ctx->in);
-                buf += 64;
-                len -= 64;
-        }
-
-        /* Handle any remaining bytes of data. */
-
-        memcpy(ctx->in, buf, len);
-}
-
-/*
- * Final wrapup - pad to 64-byte boundary with the bit pattern
- * 1 0* (64-bit count of bits processed, MSB-first)
- */
-static void MD5Final(unsigned char digest[16], MD5Context *ctx){
-        unsigned count;
-        unsigned char *p;
-
-        /* Compute number of bytes mod 64 */
-        count = (ctx->bits[0] >> 3) & 0x3F;
-
-        /* Set the first char of padding to 0x80.  This is safe since there is
-           always at least one byte free */
-        p = ctx->in + count;
-        *p++ = 0x80;
-
-        /* Bytes of padding needed to make 64 bytes */
-        count = 64 - 1 - count;
-
-        /* Pad out to 56 mod 64 */
-        if (count < 8) {
-                /* Two lots of padding:  Pad the first block to 64 bytes */
-                memset(p, 0, count);
-                byteReverse(ctx->in, 16);
-                MD5Transform(ctx->buf, (uint32 *)ctx->in);
-
-                /* Now fill the next block with 56 bytes */
-                memset(ctx->in, 0, 56);
-        } else {
-                /* Pad block to 56 bytes */
-                memset(p, 0, count-8);
-        }
-        byteReverse(ctx->in, 14);
-
-        /* Append length in bits and transform */
-        memcpy(ctx->in + 14*4, ctx->bits, 8);
-
-        MD5Transform(ctx->buf, (uint32 *)ctx->in);
-        byteReverse((unsigned char *)ctx->buf, 4);
-        memcpy(digest, ctx->buf, 16);
-}
-
-/*
-** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
-*/
-static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
-  static char const zEncode[] = "0123456789abcdef";
-  int i, j;
-
-  for(j=i=0; i<16; i++){
-    int a = digest[i];
-    zBuf[j++] = zEncode[(a>>4)&0xf];
-    zBuf[j++] = zEncode[a & 0xf];
-  }
-  zBuf[j] = 0;
-}
-
-
-/*
-** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers
-** each representing 16 bits of the digest and separated from each
-** other by a "-" character.
-*/
-static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){
-  int i, j;
-  unsigned int x;
-  for(i=j=0; i<16; i+=2){
-    x = digest[i]*256 + digest[i+1];
-    if( i>0 ) zDigest[j++] = '-';
-    sqlite3_snprintf(50-j, &zDigest[j], "%05u", x);
-    j += 5;
-  }
-  zDigest[j] = 0;
-}
-
-/*
-** A TCL command for md5.  The argument is the text to be hashed.  The
-** Result is the hash in base64.
-*/
-static int SQLITE_TCLAPI md5_cmd(
-  void*cd,
-  Tcl_Interp *interp,
-  int argc,
-  const char **argv
-){
-  MD5Context ctx;
-  unsigned char digest[16];
-  char zBuf[50];
-  void (*converter)(unsigned char*, char*);
-
-  if( argc!=2 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
-        " TEXT\"", (char*)0);
-    return TCL_ERROR;
-  }
-  MD5Init(&ctx);
-  MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1]));
-  MD5Final(digest, &ctx);
-  converter = (void(*)(unsigned char*,char*))cd;
-  converter(digest, zBuf);
-  Tcl_AppendResult(interp, zBuf, (char*)0);
-  return TCL_OK;
-}
-
-/*
-** A TCL command to take the md5 hash of a file.  The argument is the
-** name of the file.
-*/
-static int SQLITE_TCLAPI md5file_cmd(
-  void*cd,
-  Tcl_Interp *interp,
-  int argc,
-  const char **argv
-){
-  FILE *in;
-  int ofst;
-  int amt;
-  MD5Context ctx;
-  void (*converter)(unsigned char*, char*);
-  unsigned char digest[16];
-  char zBuf[10240];
-
-  if( argc!=2 && argc!=4 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
-        " FILENAME [OFFSET AMT]\"", (char*)0);
-    return TCL_ERROR;
-  }
-  if( argc==4 ){
-    ofst = atoi(argv[2]);
-    amt = atoi(argv[3]);
-  }else{
-    ofst = 0;
-    amt = 2147483647;
-  }
-  in = fopen(argv[1],"rb");
-  if( in==0 ){
-    Tcl_AppendResult(interp,"unable to open file \"", argv[1],
-         "\" for reading", (char*)0);
-    return TCL_ERROR;
-  }
-  fseek(in, ofst, SEEK_SET);
-  MD5Init(&ctx);
-  while( amt>0 ){
-    int n;
-    n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in);
-    if( n<=0 ) break;
-    MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
-    amt -= n;
-  }
-  fclose(in);
-  MD5Final(digest, &ctx);
-  converter = (void(*)(unsigned char*,char*))cd;
-  converter(digest, zBuf);
-  Tcl_AppendResult(interp, zBuf, (char*)0);
-  return TCL_OK;
-}
-
-/*
-** Register the four new TCL commands for generating MD5 checksums
-** with the TCL interpreter.
-*/
-int Md5_Init(Tcl_Interp *interp){
-  Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd,
-                    MD5DigestToBase16, 0);
-  Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd,
-                    MD5DigestToBase10x8, 0);
-  Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd,
-                    MD5DigestToBase16, 0);
-  Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd,
-                    MD5DigestToBase10x8, 0);
-  return TCL_OK;
-}
-#endif /* defined(SQLITE_TEST) || defined(SQLITE_TCLMD5) */
-
-#if defined(SQLITE_TEST)
-/*
-** During testing, the special md5sum() aggregate function is available.
-** inside SQLite.  The following routines implement that function.
-*/
-static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){
-  MD5Context *p;
-  int i;
-  if( argc<1 ) return;
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  if( p==0 ) return;
-  if( !p->isInit ){
-    MD5Init(p);
-  }
-  for(i=0; i<argc; i++){
-    const char *zData = (char*)sqlite3_value_text(argv[i]);
-    if( zData ){
-      MD5Update(p, (unsigned char*)zData, (int)strlen(zData));
-    }
-  }
-}
-static void md5finalize(sqlite3_context *context){
-  MD5Context *p;
-  unsigned char digest[16];
-  char zBuf[33];
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  MD5Final(digest,p);
-  MD5DigestToBase16(digest, zBuf);
-  sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
-}
-int Md5_Register(
-  sqlite3 *db,
-  char **pzErrMsg,
-  const sqlite3_api_routines *pThunk
-){
-  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0,
-                                 md5step, md5finalize);
-  sqlite3_overload_function(db, "md5sum", -1);  /* To exercise this API */
-  return rc;
-}
-#endif /* defined(SQLITE_TEST) */
-
 
 /*
 ** If the macro TCLSH is one, then put in code this for the
@@ -4021,30 +3594,11 @@ static const char *tclsh_main_loop(void){
 static const char *tclsh_main_loop(void);
 #endif
 
+/* The following to TCL commands used for testing must appear in this
+** file (they cannot be factored out into one of the test_*.c files where
+** they belong) because they require access to the SqliteDb object.
+*/
 #ifdef SQLITE_TEST
-static void init_all(Tcl_Interp *);
-static int SQLITE_TCLAPI init_all_cmd(
-  ClientData cd,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-
-  Tcl_Interp *slave;
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "SLAVE");
-    return TCL_ERROR;
-  }
-
-  slave = Tcl_GetSlave(interp, Tcl_GetString(objv[1]));
-  if( !slave ){
-    return TCL_ERROR;
-  }
-
-  init_all(slave);
-  return TCL_OK;
-}
-
 /*
 ** Tclcmd: db_use_legacy_prepare DB BOOLEAN
 **
@@ -4123,141 +3677,6 @@ static int SQLITE_TCLAPI db_last_stmt_ptr(
 }
 #endif /* SQLITE_TEST */
 
-/*
-** Configure the interpreter passed as the first argument to have access
-** to the commands and linked variables that make up:
-**
-**   * the [sqlite3] extension itself,
-**
-**   * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and
-**
-**   * If SQLITE_TEST is set, the various test interfaces used by the Tcl
-**     test suite.
-*/
-static void init_all(Tcl_Interp *interp){
-  Sqlite3_Init(interp);
-
-#if defined(SQLITE_TEST) || defined(SQLITE_TCLMD5)
-  Md5_Init(interp);
-#endif
-
-#ifdef SQLITE_TEST
-  {
-    extern int Sqliteconfig_Init(Tcl_Interp*);
-    extern int Sqlitetest1_Init(Tcl_Interp*);
-    extern int Sqlitetest2_Init(Tcl_Interp*);
-    extern int Sqlitetest3_Init(Tcl_Interp*);
-    extern int Sqlitetest4_Init(Tcl_Interp*);
-    extern int Sqlitetest5_Init(Tcl_Interp*);
-    extern int Sqlitetest6_Init(Tcl_Interp*);
-    extern int Sqlitetest7_Init(Tcl_Interp*);
-    extern int Sqlitetest8_Init(Tcl_Interp*);
-    extern int Sqlitetest9_Init(Tcl_Interp*);
-    extern int Sqlitetestasync_Init(Tcl_Interp*);
-    extern int Sqlitetest_autoext_Init(Tcl_Interp*);
-    extern int Sqlitetest_blob_Init(Tcl_Interp*);
-    extern int Sqlitetest_demovfs_Init(Tcl_Interp *);
-    extern int Sqlitetest_func_Init(Tcl_Interp*);
-    extern int Sqlitetest_hexio_Init(Tcl_Interp*);
-    extern int Sqlitetest_init_Init(Tcl_Interp*);
-    extern int Sqlitetest_malloc_Init(Tcl_Interp*);
-    extern int Sqlitetest_mutex_Init(Tcl_Interp*);
-    extern int Sqlitetestschema_Init(Tcl_Interp*);
-    extern int Sqlitetestsse_Init(Tcl_Interp*);
-    extern int Sqlitetesttclvar_Init(Tcl_Interp*);
-    extern int Sqlitetestfs_Init(Tcl_Interp*);
-    extern int SqlitetestThread_Init(Tcl_Interp*);
-    extern int SqlitetestOnefile_Init();
-    extern int SqlitetestOsinst_Init(Tcl_Interp*);
-    extern int Sqlitetestbackup_Init(Tcl_Interp*);
-    extern int Sqlitetestintarray_Init(Tcl_Interp*);
-    extern int Sqlitetestvfs_Init(Tcl_Interp *);
-    extern int Sqlitetestrtree_Init(Tcl_Interp*);
-    extern int Sqlitequota_Init(Tcl_Interp*);
-    extern int Sqlitemultiplex_Init(Tcl_Interp*);
-    extern int SqliteSuperlock_Init(Tcl_Interp*);
-    extern int SqlitetestSyscall_Init(Tcl_Interp*);
-#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
-    extern int TestSession_Init(Tcl_Interp*);
-#endif
-    extern int Fts5tcl_Init(Tcl_Interp *);
-    extern int SqliteRbu_Init(Tcl_Interp*);
-    extern int Sqlitetesttcl_Init(Tcl_Interp*);
-#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
-    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
-#endif
-
-#ifdef SQLITE_ENABLE_ZIPVFS
-    extern int Zipvfs_Init(Tcl_Interp*);
-    Zipvfs_Init(interp);
-#endif
-
-    Sqliteconfig_Init(interp);
-    Sqlitetest1_Init(interp);
-    Sqlitetest2_Init(interp);
-    Sqlitetest3_Init(interp);
-    Sqlitetest4_Init(interp);
-    Sqlitetest5_Init(interp);
-    Sqlitetest6_Init(interp);
-    Sqlitetest7_Init(interp);
-    Sqlitetest8_Init(interp);
-    Sqlitetest9_Init(interp);
-    Sqlitetestasync_Init(interp);
-    Sqlitetest_autoext_Init(interp);
-    Sqlitetest_blob_Init(interp);
-    Sqlitetest_demovfs_Init(interp);
-    Sqlitetest_func_Init(interp);
-    Sqlitetest_hexio_Init(interp);
-    Sqlitetest_init_Init(interp);
-    Sqlitetest_malloc_Init(interp);
-    Sqlitetest_mutex_Init(interp);
-    Sqlitetestschema_Init(interp);
-    Sqlitetesttclvar_Init(interp);
-    Sqlitetestfs_Init(interp);
-    SqlitetestThread_Init(interp);
-    SqlitetestOnefile_Init();
-    SqlitetestOsinst_Init(interp);
-    Sqlitetestbackup_Init(interp);
-    Sqlitetestintarray_Init(interp);
-    Sqlitetestvfs_Init(interp);
-    Sqlitetestrtree_Init(interp);
-    Sqlitequota_Init(interp);
-    Sqlitemultiplex_Init(interp);
-    SqliteSuperlock_Init(interp);
-    SqlitetestSyscall_Init(interp);
-#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
-    TestSession_Init(interp);
-#endif
-    Fts5tcl_Init(interp);
-    SqliteRbu_Init(interp);
-    Sqlitetesttcl_Init(interp);
-
-#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
-    Sqlitetestfts3_Init(interp);
-#endif
-
-    Tcl_CreateObjCommand(
-        interp, "load_testfixture_extensions", init_all_cmd, 0, 0
-    );
-    Tcl_CreateObjCommand(
-        interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0
-    );
-    Tcl_CreateObjCommand(
-        interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0
-    );
-
-#ifdef SQLITE_SSE
-    Sqlitetestsse_Init(interp);
-#endif
-  }
-#endif
-}
-
-/* Needed for the setrlimit() system call on unix */
-#if defined(unix)
-#include <sys/resource.h>
-#endif
-
 #define TCLSH_MAIN main   /* Needed to fake out mktclapp */
 int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_Interp *interp;
@@ -4271,17 +3690,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   }
 #endif
 
-  /* Since the primary use case for this binary is testing of SQLite,
-  ** be sure to generate core files if we crash */
-#if defined(SQLITE_TEST) && defined(unix)
-  { struct rlimit x;
-    getrlimit(RLIMIT_CORE, &x);
-    x.rlim_cur = x.rlim_max;
-    setrlimit(RLIMIT_CORE, &x);
-  }
-#endif /* SQLITE_TEST && unix */
-
-
   /* Call sqlite3_shutdown() once before doing anything else. This is to
   ** test that sqlite3_shutdown() can be safely called by a process before
   ** sqlite3_initialize() is. */
@@ -4295,7 +3703,23 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
 #endif
 
-  init_all(interp);
+  /* Add extensions */
+#if !defined(SQLITE_TEST)
+  /* Normally we only initialize the TCL extension */
+  Sqlite3_Init(interp);
+#else
+  /* For testing, do lots of extra initialization */
+  {
+    extern void sqlite3InitTclTestLogic(Tcl_Interp*);
+    sqlite3InitTclTestLogic(interp);
+    Tcl_CreateObjCommand(
+        interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0
+    );
+    Tcl_CreateObjCommand(
+        interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0
+    );
+  }
+#endif /* SQLITE_TEST */
   if( argc>=2 ){
     int i;
     char zArgc[32];
diff --git a/src/test_md5.c b/src/test_md5.c
new file mode 100644
index 0000000000..b670026861
--- /dev/null
+++ b/src/test_md5.c
@@ -0,0 +1,450 @@
+/*
+** 2017-10-13
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains code to implement an MD5 extension to TCL.
+*/
+#include "sqlite3.h"
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+# endif
+#endif
+
+/*
+ * This code implements the MD5 message-digest algorithm.
+ * The algorithm is due to Ron Rivest.  This code was
+ * written by Colin Plumb in 1993, no copyright is claimed.
+ * This code is in the public domain; do with it what you wish.
+ *
+ * Equivalent code is available from RSA Data Security, Inc.
+ * This code has been tested against that, and is equivalent,
+ * except that you don't need to include two pages of legalese
+ * with every copy.
+ *
+ * To compute the message digest of a chunk of bytes, declare an
+ * MD5Context structure, pass it to MD5Init, call MD5Update as
+ * needed on buffers full of bytes, and then call MD5Final, which
+ * will fill a supplied 16-byte array with the digest.
+ */
+
+/*
+ * If compiled on a machine that doesn't have a 32-bit integer,
+ * you just set "uint32" to the appropriate datatype for an
+ * unsigned 32-bit integer.  For example:
+ *
+ *       cc -Duint32='unsigned long' md5.c
+ *
+ */
+#ifndef uint32
+#  define uint32 unsigned int
+#endif
+
+struct MD5Context {
+  int isInit;
+  uint32 buf[4];
+  uint32 bits[2];
+  unsigned char in[64];
+};
+typedef struct MD5Context MD5Context;
+
+/*
+ * Note: this code is harmless on little-endian machines.
+ */
+static void byteReverse (unsigned char *buf, unsigned longs){
+        uint32 t;
+        do {
+                t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 |
+                            ((unsigned)buf[1]<<8 | buf[0]);
+                *(uint32 *)buf = t;
+                buf += 4;
+        } while (--longs);
+}
+/* The four core functions - F1 is optimized somewhat */
+
+/* #define F1(x, y, z) (x & y | ~x & z) */
+#define F1(x, y, z) (z ^ (x & (y ^ z)))
+#define F2(x, y, z) F1(z, x, y)
+#define F3(x, y, z) (x ^ y ^ z)
+#define F4(x, y, z) (y ^ (x | ~z))
+
+/* This is the central step in the MD5 algorithm. */
+#define MD5STEP(f, w, x, y, z, data, s) \
+        ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
+
+/*
+ * The core of the MD5 algorithm, this alters an existing MD5 hash to
+ * reflect the addition of 16 longwords of new data.  MD5Update blocks
+ * the data and converts bytes into longwords for this routine.
+ */
+static void MD5Transform(uint32 buf[4], const uint32 in[16]){
+        register uint32 a, b, c, d;
+
+        a = buf[0];
+        b = buf[1];
+        c = buf[2];
+        d = buf[3];
+
+        MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478,  7);
+        MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
+        MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
+        MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
+        MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf,  7);
+        MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
+        MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
+        MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
+        MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8,  7);
+        MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
+        MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
+        MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
+        MD5STEP(F1, a, b, c, d, in[12]+0x6b901122,  7);
+        MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
+        MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
+        MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
+
+        MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562,  5);
+        MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340,  9);
+        MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
+        MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
+        MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d,  5);
+        MD5STEP(F2, d, a, b, c, in[10]+0x02441453,  9);
+        MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
+        MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
+        MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6,  5);
+        MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6,  9);
+        MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
+        MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
+        MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905,  5);
+        MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8,  9);
+        MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
+        MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
+
+        MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942,  4);
+        MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
+        MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
+        MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
+        MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44,  4);
+        MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
+        MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
+        MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
+        MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6,  4);
+        MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
+        MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
+        MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
+        MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039,  4);
+        MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
+        MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
+        MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
+
+        MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244,  6);
+        MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
+        MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
+        MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
+        MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3,  6);
+        MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
+        MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
+        MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
+        MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f,  6);
+        MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
+        MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
+        MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
+        MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82,  6);
+        MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
+        MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
+        MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
+
+        buf[0] += a;
+        buf[1] += b;
+        buf[2] += c;
+        buf[3] += d;
+}
+
+/*
+ * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
+ * initialization constants.
+ */
+static void MD5Init(MD5Context *ctx){
+        ctx->isInit = 1;
+        ctx->buf[0] = 0x67452301;
+        ctx->buf[1] = 0xefcdab89;
+        ctx->buf[2] = 0x98badcfe;
+        ctx->buf[3] = 0x10325476;
+        ctx->bits[0] = 0;
+        ctx->bits[1] = 0;
+}
+
+/*
+ * Update context to reflect the concatenation of another buffer full
+ * of bytes.
+ */
+static
+void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
+        uint32 t;
+
+        /* Update bitcount */
+
+        t = ctx->bits[0];
+        if ((ctx->bits[0] = t + ((uint32)len << 3)) < t)
+                ctx->bits[1]++; /* Carry from low to high */
+        ctx->bits[1] += len >> 29;
+
+        t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
+
+        /* Handle any leading odd-sized chunks */
+
+        if ( t ) {
+                unsigned char *p = (unsigned char *)ctx->in + t;
+
+                t = 64-t;
+                if (len < t) {
+                        memcpy(p, buf, len);
+                        return;
+                }
+                memcpy(p, buf, t);
+                byteReverse(ctx->in, 16);
+                MD5Transform(ctx->buf, (uint32 *)ctx->in);
+                buf += t;
+                len -= t;
+        }
+
+        /* Process data in 64-byte chunks */
+
+        while (len >= 64) {
+                memcpy(ctx->in, buf, 64);
+                byteReverse(ctx->in, 16);
+                MD5Transform(ctx->buf, (uint32 *)ctx->in);
+                buf += 64;
+                len -= 64;
+        }
+
+        /* Handle any remaining bytes of data. */
+
+        memcpy(ctx->in, buf, len);
+}
+
+/*
+ * Final wrapup - pad to 64-byte boundary with the bit pattern
+ * 1 0* (64-bit count of bits processed, MSB-first)
+ */
+static void MD5Final(unsigned char digest[16], MD5Context *ctx){
+        unsigned count;
+        unsigned char *p;
+
+        /* Compute number of bytes mod 64 */
+        count = (ctx->bits[0] >> 3) & 0x3F;
+
+        /* Set the first char of padding to 0x80.  This is safe since there is
+           always at least one byte free */
+        p = ctx->in + count;
+        *p++ = 0x80;
+
+        /* Bytes of padding needed to make 64 bytes */
+        count = 64 - 1 - count;
+
+        /* Pad out to 56 mod 64 */
+        if (count < 8) {
+                /* Two lots of padding:  Pad the first block to 64 bytes */
+                memset(p, 0, count);
+                byteReverse(ctx->in, 16);
+                MD5Transform(ctx->buf, (uint32 *)ctx->in);
+
+                /* Now fill the next block with 56 bytes */
+                memset(ctx->in, 0, 56);
+        } else {
+                /* Pad block to 56 bytes */
+                memset(p, 0, count-8);
+        }
+        byteReverse(ctx->in, 14);
+
+        /* Append length in bits and transform */
+        memcpy(ctx->in + 14*4, ctx->bits, 8);
+
+        MD5Transform(ctx->buf, (uint32 *)ctx->in);
+        byteReverse((unsigned char *)ctx->buf, 4);
+        memcpy(digest, ctx->buf, 16);
+}
+
+/*
+** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
+*/
+static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
+  static char const zEncode[] = "0123456789abcdef";
+  int i, j;
+
+  for(j=i=0; i<16; i++){
+    int a = digest[i];
+    zBuf[j++] = zEncode[(a>>4)&0xf];
+    zBuf[j++] = zEncode[a & 0xf];
+  }
+  zBuf[j] = 0;
+}
+
+
+/*
+** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers
+** each representing 16 bits of the digest and separated from each
+** other by a "-" character.
+*/
+static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){
+  int i, j;
+  unsigned int x;
+  for(i=j=0; i<16; i+=2){
+    x = digest[i]*256 + digest[i+1];
+    if( i>0 ) zDigest[j++] = '-';
+    sqlite3_snprintf(50-j, &zDigest[j], "%05u", x);
+    j += 5;
+  }
+  zDigest[j] = 0;
+}
+
+/*
+** A TCL command for md5.  The argument is the text to be hashed.  The
+** Result is the hash in base64.
+*/
+static int SQLITE_TCLAPI md5_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
+  MD5Context ctx;
+  unsigned char digest[16];
+  char zBuf[50];
+  void (*converter)(unsigned char*, char*);
+
+  if( argc!=2 ){
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
+        " TEXT\"", (char*)0);
+    return TCL_ERROR;
+  }
+  MD5Init(&ctx);
+  MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1]));
+  MD5Final(digest, &ctx);
+  converter = (void(*)(unsigned char*,char*))cd;
+  converter(digest, zBuf);
+  Tcl_AppendResult(interp, zBuf, (char*)0);
+  return TCL_OK;
+}
+
+/*
+** A TCL command to take the md5 hash of a file.  The argument is the
+** name of the file.
+*/
+static int SQLITE_TCLAPI md5file_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
+  FILE *in;
+  int ofst;
+  int amt;
+  MD5Context ctx;
+  void (*converter)(unsigned char*, char*);
+  unsigned char digest[16];
+  char zBuf[10240];
+
+  if( argc!=2 && argc!=4 ){
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
+        " FILENAME [OFFSET AMT]\"", (char*)0);
+    return TCL_ERROR;
+  }
+  if( argc==4 ){
+    ofst = atoi(argv[2]);
+    amt = atoi(argv[3]);
+  }else{
+    ofst = 0;
+    amt = 2147483647;
+  }
+  in = fopen(argv[1],"rb");
+  if( in==0 ){
+    Tcl_AppendResult(interp,"unable to open file \"", argv[1],
+         "\" for reading", (char*)0);
+    return TCL_ERROR;
+  }
+  fseek(in, ofst, SEEK_SET);
+  MD5Init(&ctx);
+  while( amt>0 ){
+    int n;
+    n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in);
+    if( n<=0 ) break;
+    MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
+    amt -= n;
+  }
+  fclose(in);
+  MD5Final(digest, &ctx);
+  converter = (void(*)(unsigned char*,char*))cd;
+  converter(digest, zBuf);
+  Tcl_AppendResult(interp, zBuf, (char*)0);
+  return TCL_OK;
+}
+
+/*
+** Register the four new TCL commands for generating MD5 checksums
+** with the TCL interpreter.
+*/
+int Md5_Init(Tcl_Interp *interp){
+  Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd,
+                    MD5DigestToBase16, 0);
+  Tcl_CreateCommand(interp, "md5-10x8", (Tcl_CmdProc*)md5_cmd,
+                    MD5DigestToBase10x8, 0);
+  Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd,
+                    MD5DigestToBase16, 0);
+  Tcl_CreateCommand(interp, "md5file-10x8", (Tcl_CmdProc*)md5file_cmd,
+                    MD5DigestToBase10x8, 0);
+  return TCL_OK;
+}
+
+/*
+** During testing, the special md5sum() aggregate function is available.
+** inside SQLite.  The following routines implement that function.
+*/
+static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){
+  MD5Context *p;
+  int i;
+  if( argc<1 ) return;
+  p = sqlite3_aggregate_context(context, sizeof(*p));
+  if( p==0 ) return;
+  if( !p->isInit ){
+    MD5Init(p);
+  }
+  for(i=0; i<argc; i++){
+    const char *zData = (char*)sqlite3_value_text(argv[i]);
+    if( zData ){
+      MD5Update(p, (unsigned char*)zData, (int)strlen(zData));
+    }
+  }
+}
+static void md5finalize(sqlite3_context *context){
+  MD5Context *p;
+  unsigned char digest[16];
+  char zBuf[33];
+  p = sqlite3_aggregate_context(context, sizeof(*p));
+  MD5Final(digest,p);
+  MD5DigestToBase16(digest, zBuf);
+  sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+}
+int Md5_Register(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pThunk
+){
+  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0,
+                                 md5step, md5finalize);
+  sqlite3_overload_function(db, "md5sum", -1);  /* To exercise this API */
+  return rc;
+}
diff --git a/src/test_tclsh.c b/src/test_tclsh.c
new file mode 100644
index 0000000000..213f5b1fe1
--- /dev/null
+++ b/src/test_tclsh.c
@@ -0,0 +1,196 @@
+/*
+** 2017-10-13
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains extensions to the the "tclsqlite.c" module used for
+** testing.  Basically, all of the other "test_*.c" modules are linked
+** into the enhanced tclsh used for testing (and named "testfixture" or
+** "testfixture.exe") using logic encoded by this file.
+**
+** The code in this file used to be found in tclsqlite3.c, contained within
+** #if SQLITE_TEST ... #endif.  It is factored out into this separate module
+** in an effort to keep the tclsqlite.c file pure.
+*/
+#include "sqlite3.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+# endif
+#endif
+
+/* Needed for the setrlimit() system call on unix */
+#if defined(unix)
+#include <sys/resource.h>
+#endif
+
+/* Forward declaration */
+static int SQLITE_TCLAPI load_testfixture_extensions(
+  ClientData cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+);
+
+/*
+** This routine is the primary export of this file.
+**
+** Configure the interpreter passed as the first argument to have access
+** to the commands and linked variables that make up:
+**
+**   * the [sqlite3] extension itself,
+**
+**   * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and
+**
+**   * If SQLITE_TEST is set, the various test interfaces used by the Tcl
+**     test suite.
+*/
+void sqlite3InitTclTestLogic(Tcl_Interp *interp){
+  /* Since the primary use case for this binary is testing of SQLite,
+  ** be sure to generate core files if we crash */
+#if defined(unix)
+  { struct rlimit x;
+    getrlimit(RLIMIT_CORE, &x);
+    x.rlim_cur = x.rlim_max;
+    setrlimit(RLIMIT_CORE, &x);
+  }
+#endif /* unix */
+
+  {
+    extern int Sqlite3_Init(Tcl_Interp*);
+    extern int Sqliteconfig_Init(Tcl_Interp*);
+    extern int Sqlitetest1_Init(Tcl_Interp*);
+    extern int Sqlitetest2_Init(Tcl_Interp*);
+    extern int Sqlitetest3_Init(Tcl_Interp*);
+    extern int Sqlitetest4_Init(Tcl_Interp*);
+    extern int Sqlitetest5_Init(Tcl_Interp*);
+    extern int Sqlitetest6_Init(Tcl_Interp*);
+    extern int Sqlitetest7_Init(Tcl_Interp*);
+    extern int Sqlitetest8_Init(Tcl_Interp*);
+    extern int Sqlitetest9_Init(Tcl_Interp*);
+    extern int Sqlitetestasync_Init(Tcl_Interp*);
+    extern int Sqlitetest_autoext_Init(Tcl_Interp*);
+    extern int Sqlitetest_blob_Init(Tcl_Interp*);
+    extern int Sqlitetest_demovfs_Init(Tcl_Interp *);
+    extern int Sqlitetest_func_Init(Tcl_Interp*);
+    extern int Sqlitetest_hexio_Init(Tcl_Interp*);
+    extern int Sqlitetest_init_Init(Tcl_Interp*);
+    extern int Sqlitetest_malloc_Init(Tcl_Interp*);
+    extern int Sqlitetest_mutex_Init(Tcl_Interp*);
+    extern int Sqlitetestschema_Init(Tcl_Interp*);
+    extern int Sqlitetestsse_Init(Tcl_Interp*);
+    extern int Sqlitetesttclvar_Init(Tcl_Interp*);
+    extern int Sqlitetestfs_Init(Tcl_Interp*);
+    extern int SqlitetestThread_Init(Tcl_Interp*);
+    extern int SqlitetestOnefile_Init();
+    extern int SqlitetestOsinst_Init(Tcl_Interp*);
+    extern int Sqlitetestbackup_Init(Tcl_Interp*);
+    extern int Sqlitetestintarray_Init(Tcl_Interp*);
+    extern int Sqlitetestvfs_Init(Tcl_Interp *);
+    extern int Sqlitetestrtree_Init(Tcl_Interp*);
+    extern int Sqlitequota_Init(Tcl_Interp*);
+    extern int Sqlitemultiplex_Init(Tcl_Interp*);
+    extern int SqliteSuperlock_Init(Tcl_Interp*);
+    extern int SqlitetestSyscall_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    extern int TestSession_Init(Tcl_Interp*);
+#endif
+    extern int Md5_Init(Tcl_Interp*);
+    extern int Fts5tcl_Init(Tcl_Interp *);
+    extern int SqliteRbu_Init(Tcl_Interp*);
+    extern int Sqlitetesttcl_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
+    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
+#endif
+#ifdef SQLITE_ENABLE_ZIPVFS
+    extern int Zipvfs_Init(Tcl_Interp*);
+#endif
+
+    Sqlite3_Init(interp);
+#ifdef SQLITE_ENABLE_ZIPVFS
+    Zipvfs_Init(interp);
+#endif
+    Md5_Init(interp);
+    Sqliteconfig_Init(interp);
+    Sqlitetest1_Init(interp);
+    Sqlitetest2_Init(interp);
+    Sqlitetest3_Init(interp);
+    Sqlitetest4_Init(interp);
+    Sqlitetest5_Init(interp);
+    Sqlitetest6_Init(interp);
+    Sqlitetest7_Init(interp);
+    Sqlitetest8_Init(interp);
+    Sqlitetest9_Init(interp);
+    Sqlitetestasync_Init(interp);
+    Sqlitetest_autoext_Init(interp);
+    Sqlitetest_blob_Init(interp);
+    Sqlitetest_demovfs_Init(interp);
+    Sqlitetest_func_Init(interp);
+    Sqlitetest_hexio_Init(interp);
+    Sqlitetest_init_Init(interp);
+    Sqlitetest_malloc_Init(interp);
+    Sqlitetest_mutex_Init(interp);
+    Sqlitetestschema_Init(interp);
+    Sqlitetesttclvar_Init(interp);
+    Sqlitetestfs_Init(interp);
+    SqlitetestThread_Init(interp);
+    SqlitetestOnefile_Init();
+    SqlitetestOsinst_Init(interp);
+    Sqlitetestbackup_Init(interp);
+    Sqlitetestintarray_Init(interp);
+    Sqlitetestvfs_Init(interp);
+    Sqlitetestrtree_Init(interp);
+    Sqlitequota_Init(interp);
+    Sqlitemultiplex_Init(interp);
+    SqliteSuperlock_Init(interp);
+    SqlitetestSyscall_Init(interp);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    TestSession_Init(interp);
+#endif
+    Fts5tcl_Init(interp);
+    SqliteRbu_Init(interp);
+    Sqlitetesttcl_Init(interp);
+
+#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
+    Sqlitetestfts3_Init(interp);
+#endif
+
+    Tcl_CreateObjCommand(
+        interp, "load_testfixture_extensions", load_testfixture_extensions,0,0
+    );
+  }
+}
+
+/* tclcmd:   load_testfixture_extensions
+*/
+static int SQLITE_TCLAPI load_testfixture_extensions(
+  ClientData cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+
+  Tcl_Interp *slave;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SLAVE");
+    return TCL_ERROR;
+  }
+
+  slave = Tcl_GetSlave(interp, Tcl_GetString(objv[1]));
+  if( !slave ){
+    return TCL_ERROR;
+  }
+
+  sqlite3InitTclTestLogic(slave);
+  return TCL_OK;
+}

From 1df6470d359e9e9cc4eacbd1932dce5354b8b18a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 13 Oct 2017 15:56:26 +0000
Subject: [PATCH 186/270] When compiling with SQLITE_TEST, extra options are
 available on the "DB version" command for controlling the internal state of
 the SqliteDb object, for testing purposes.

FossilOrigin-Name: bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896
---
 Makefile.in          |   1 +
 Makefile.msc         |   1 +
 main.mk              |   1 +
 manifest             |  23 ++++----
 manifest.uuid        |   2 +-
 src/tclsqlite.c      | 126 +++++++++++++------------------------------
 test/scanstatus.test |   4 +-
 7 files changed, 52 insertions(+), 106 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 8c306629e0..d474222fb8 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1119,6 +1119,7 @@ TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
diff --git a/Makefile.msc b/Makefile.msc
index e4a72e872b..0611d8a8b3 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2111,6 +2111,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
diff --git a/main.mk b/main.mk
index 84d1a3eb76..bdbb56e229 100644
--- a/main.mk
+++ b/main.mk
@@ -803,6 +803,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
+TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
 
 testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
 	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
diff --git a/manifest b/manifest
index 250e0c8001..74a206741c 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Move\ssome\stest\slogic\sout\sof\stclsqlite.c\sand\sinto\sauxiliary\stest_*.c\sfiles.\nThis\sis\sa\swork\sin\sprogress.
-D 2017-10-13T15:06:06.783
-F Makefile.in 8658943abe79d10adfed7b504936199ebcd5c1d372d176936d5edd075492b13b
+C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes.
+D 2017-10-13T15:56:26.514
+F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 22fc2be4ac777553096de774b9b601568fa21904d1d64f595916f78957025558
+F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 03a3ae65678b238a787f0aa9162e0e2bd998561bc8d66b9b073f54d9025dee61
+F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 93b2267ade2bfaad9d4f0135e5928ee28fdd77a355a563ad444ead13e1c488b2
+F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4
 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
@@ -1157,7 +1157,7 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
 F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa
 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
-F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285
+F test/scanstatus.test 1ba0e2ee25dcd1d55ec770803b19832cffaecbf0b15d376807759ebeed3669b0
 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
@@ -1663,10 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a
-R 225e4cd689e0c61d406db3daa52debdc
-T *branch * tclsqlite-cleanup
-T *sym-tclsqlite-cleanup *
-T -sym-trunk *
+P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf
+R beaa5ad001aa19fb6dcaa6d5d2766f78
 U drh
-Z 604c544fbf5b9e259c249448beb617c7
+Z 978089b1abbc5f3c189d1cdd601af59d
diff --git a/manifest.uuid b/manifest.uuid
index 705f05a85e..ddda81051f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf
\ No newline at end of file
+bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index a66fe6832e..0009eab695 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -3286,7 +3286,42 @@ static int SQLITE_TCLAPI DbObjCmd(
   ** Return the version string for this database.
   */
   case DB_VERSION: {
-    Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC);
+    int i;
+    for(i=2; i<objc; i++){
+      const char *zArg = Tcl_GetString(objv[i]);
+      /* Optional arguments to $db version are used for testing purpose */
+#ifdef SQLITE_TEST
+      /* $db version -use-legacy-prepare BOOLEAN
+      **
+      ** Turn the use of legacy sqlite3_prepare() on or off.
+      */
+      if( strcmp(zArg, "-use-legacy-prepare")==0 && i+1<objc ){
+        i++;
+        if( Tcl_GetBooleanFromObj(interp, objv[i], &pDb->bLegacyPrepare) ){
+          return TCL_ERROR;
+        }
+      }else
+
+      /* $db version -last-stmt-ptr
+      **
+      ** Return a string which is a hex encoding of the pointer to the
+      ** most recent sqlite3_stmt in the statement cache.
+      */
+      if( strcmp(zArg, "-last-stmt-ptr")==0 ){
+        char zBuf[100];
+        sqlite3_snprintf(sizeof(zBuf), zBuf, "%p",
+                         pDb->stmtList ? pDb->stmtList->pStmt: 0);
+        Tcl_SetResult(interp, zBuf, TCL_VOLATILE);
+      }else
+#endif /* SQLITE_TEST */
+      {
+        Tcl_AppendResult(interp, "unknown argument: ", zArg, (char*)0);
+        return TCL_ERROR;
+      }
+    }
+    if( i==2 ){   
+      Tcl_SetResult(interp, (char *)sqlite3_libversion(), TCL_STATIC);
+    }
     break;
   }
 
@@ -3594,89 +3629,6 @@ static const char *tclsh_main_loop(void){
 static const char *tclsh_main_loop(void);
 #endif
 
-/* The following to TCL commands used for testing must appear in this
-** file (they cannot be factored out into one of the test_*.c files where
-** they belong) because they require access to the SqliteDb object.
-*/
-#ifdef SQLITE_TEST
-/*
-** Tclcmd: db_use_legacy_prepare DB BOOLEAN
-**
-**   The first argument to this command must be a database command created by
-**   [sqlite3]. If the second argument is true, then the handle is configured
-**   to use the sqlite3_prepare_v2() function to prepare statements. If it
-**   is false, sqlite3_prepare().
-*/
-static int SQLITE_TCLAPI db_use_legacy_prepare_cmd(
-  ClientData cd,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  Tcl_CmdInfo cmdInfo;
-  SqliteDb *pDb;
-  int bPrepare;
-
-  if( objc!=3 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN");
-    return TCL_ERROR;
-  }
-
-  if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){
-    Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0);
-    return TCL_ERROR;
-  }
-  pDb = (SqliteDb*)cmdInfo.objClientData;
-  if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){
-    return TCL_ERROR;
-  }
-
-  pDb->bLegacyPrepare = bPrepare;
-
-  Tcl_ResetResult(interp);
-  return TCL_OK;
-}
-
-/*
-** Tclcmd: db_last_stmt_ptr DB
-**
-**   If the statement cache associated with database DB is not empty,
-**   return the text representation of the most recently used statement
-**   handle.
-*/
-static int SQLITE_TCLAPI db_last_stmt_ptr(
-  ClientData cd,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*);
-  Tcl_CmdInfo cmdInfo;
-  SqliteDb *pDb;
-  sqlite3_stmt *pStmt = 0;
-  char zBuf[100];
-
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB");
-    return TCL_ERROR;
-  }
-
-  if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){
-    Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0);
-    return TCL_ERROR;
-  }
-  pDb = (SqliteDb*)cmdInfo.objClientData;
-
-  if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt;
-  if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){
-    return TCL_ERROR;
-  }
-  Tcl_SetResult(interp, zBuf, TCL_VOLATILE);
-
-  return TCL_OK;
-}
-#endif /* SQLITE_TEST */
-
 #define TCLSH_MAIN main   /* Needed to fake out mktclapp */
 int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_Interp *interp;
@@ -3712,12 +3664,6 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   {
     extern void sqlite3InitTclTestLogic(Tcl_Interp*);
     sqlite3InitTclTestLogic(interp);
-    Tcl_CreateObjCommand(
-        interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0
-    );
-    Tcl_CreateObjCommand(
-        interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0
-    );
   }
 #endif /* SQLITE_TEST */
   if( argc>=2 ){
diff --git a/test/scanstatus.test b/test/scanstatus.test
index ed24d97437..fdd6476ae9 100644
--- a/test/scanstatus.test
+++ b/test/scanstatus.test
@@ -30,7 +30,7 @@ do_execsql_test 1.0 {
 }
 
 proc do_scanstatus_test {tn res} {
-  set stmt [db_last_stmt_ptr db]
+  set stmt [db version -last-stmt-ptr]
   set idx 0
   set ret [list]
   while {1} {
@@ -79,7 +79,7 @@ do_scanstatus_test 1.9 {
 }
 
 do_test 1.9 {
-  sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db]
+  sqlite3_stmt_scanstatus_reset [db version -last-stmt-ptr]
 } {}
 
 do_scanstatus_test 1.10 {

From 8544d43e4ae349ab04aea9fe6c075627abaafeeb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 13 Oct 2017 16:01:36 +0000
Subject: [PATCH 187/270] The db_use_legacy_prepare command no longer exists. 
 Use the undocumented -use-legacy-prepare option to DB version instead.

FossilOrigin-Name: ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b
---
 manifest               | 12 ++++++------
 manifest.uuid          |  2 +-
 test/permutations.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 74a206741c..cc4d9453b6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\scompiling\swith\sSQLITE_TEST,\sextra\soptions\sare\savailable\son\sthe\n"DB\sversion"\scommand\sfor\scontrolling\sthe\sinternal\sstate\sof\sthe\sSqliteDb\nobject,\sfor\stesting\spurposes.
-D 2017-10-13T15:56:26.514
+C The\sdb_use_legacy_prepare\scommand\sno\slonger\sexists.\s\sUse\sthe\sundocumented\n-use-legacy-prepare\soption\sto\sDB\sversion\sinstead.
+D 2017-10-13T16:01:36.071
 F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5
@@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test d911c9ba49088d22054a05dc73743f677872a92ac89288bcdeafa0ebf3f9c531
+F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6
 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1663,7 +1663,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 95b7687fed75b32a62a0c62d397f4f543bf40095e13c22e15938d5dcfd71fcdf
-R beaa5ad001aa19fb6dcaa6d5d2766f78
+P bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896
+R ef5c1b585866b0d225763ce3d0edba7c
 U drh
-Z 978089b1abbc5f3c189d1cdd601af59d
+Z e1aad8bb9151be888a26c10aa66d2e50
diff --git a/manifest.uuid b/manifest.uuid
index ddda81051f..5b87ac07a6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf86478d9c6b899d6441e80cf9d17b58731e5d354e31b11f0d58a798f6fab896
\ No newline at end of file
+ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b
\ No newline at end of file
diff --git a/test/permutations.test b/test/permutations.test
index 5afc51cb7d..1e04265668 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -1065,7 +1065,7 @@ test_suite "no_optimization" -description {
 test_suite "prepare" -description {
   Run tests with the db connection using sqlite3_prepare() instead of _v2().
 } -dbconfig {
-  db_use_legacy_prepare $::dbhandle 1
+  $::dbhandle version -use-legacy-prepare 1
   #$::dbhandle cache size 0
 } -files [
   test_set $allquicktests -exclude *malloc* *ioerr* *fault* \

From 903b23022d6d39fbc18d619dfdead39277bed72d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 13 Oct 2017 18:58:55 +0000
Subject: [PATCH 188/270] Add the tcl/mkccode.tcl script used to construct a
 single C-language source fiel for programs that combine C-code, SQLite, and
 TCL.  Use this script to construct the sqlite3_analyzer program.

FossilOrigin-Name: 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82
---
 Makefile.in                |  10 +-
 Makefile.msc               |  10 +-
 main.mk                    |  10 +-
 manifest                   |  20 +-
 manifest.uuid              |   2 +-
 tool/mkccode.tcl           |  93 ++++
 tool/sqlite3_analyzer.c.in | 903 +++++++++++++++++++++++++++++++++++++
 tool/tostr.tcl             |  12 -
 8 files changed, 1013 insertions(+), 47 deletions(-)
 create mode 100755 tool/mkccode.tcl
 create mode 100644 tool/sqlite3_analyzer.c.in
 delete mode 100644 tool/tostr.tcl

diff --git a/Makefile.in b/Makefile.in
index d474222fb8..e2934d80b1 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1180,14 +1180,8 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
-	echo "; return zMainloop; }" >> $@
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
diff --git a/Makefile.msc b/Makefile.msc
index 0611d8a8b3..bfd368c558 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2191,14 +2191,8 @@ smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
 
-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
-	echo #define TCLSH 2 > $@
-	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
-	echo static const char *tclsh_main_loop(void){ >> $@
-	echo static const char *zMainloop = >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
-	echo ; return zMainloop; } >> $@
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
 
 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
diff --git a/main.mk b/main.mk
index bdbb56e229..18d7cd3459 100644
--- a/main.mk
+++ b/main.mk
@@ -780,14 +780,8 @@ tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
 	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
 		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
-	echo "; return zMainloop; }" >> $@
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl
+	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
diff --git a/manifest b/manifest
index c45a7ce09e..669c1b284a 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Move\sa\sbunch\sof\sunrelated\stest\scode\sout\sof\stclsqlite.c\sand\sinto\ntest_*.c\sfiles.\s\sThere\sis\sstill\ssome\stest\scode\sin\stclsqlite.c,\sbut\sthe\samount\nis\sgreatly\sreduced.
-D 2017-10-13T16:19:48.503
-F Makefile.in d9acf9cc5252f406c2a417c12475ef14fe49096d121bd025a230fe3b58e0f294
+C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram.
+D 2017-10-13T18:58:55.897
+F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc c20c8f2945955931d0f790104b335b37ba511eeb0d75eded23b7f878077ee1f5
+F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 9311f1d83efd6818b7fbeb1256dbc5377159359528be4e7d925345e4d80446bf
+F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1593,6 +1593,7 @@ F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7
+F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c
 F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4
@@ -1631,11 +1632,11 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
+F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148
-F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4
 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003
 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c
 F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
@@ -1663,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P eaeeb09d4aa1dbccdd2488af8461e2a8c8a53d92c63fd56330be041ad72a9e4a ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b
-R ef5c1b585866b0d225763ce3d0edba7c
-T +closed ea2fa9ab9b3c97e001c1a2f3e24f3554b544070d5f915b9827eb07eb5ea8f00b
+P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d
+R 78f34b33921be1c7a7c8587f43b33b81
 U drh
-Z 0c47718eb3eab9783f8c2bca1833cb14
+Z 180ea09ee34fefc9e97e7461c13719ef
diff --git a/manifest.uuid b/manifest.uuid
index 05172a0f2b..fabaeec3c0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d
\ No newline at end of file
+298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82
\ No newline at end of file
diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl
new file mode 100755
index 0000000000..41b09f1e81
--- /dev/null
+++ b/tool/mkccode.tcl
@@ -0,0 +1,93 @@
+#!/usr/bin/tclsh
+#
+# Use this script to build C-language source code for a program that uses
+# tclsqlite.c together with custom TCL scripts and/or C extensions for
+# either SQLite or TCL.
+#
+# Usage example:
+#
+#     tclsh mktclsqliteprog.tcl demoapp.c.in >demoapp.c
+#
+# The demoapp.c.in file contains a mixture of C code, TCL script, and
+# processing directives used by mktclsqliteprog.tcl to build the final C-code
+# output file.  Most lines of demoapp.c.in are copied straight through into
+# the output.  The following control directives are recognized:
+#
+# BEGIN_STRING
+#
+#      This marks the beginning of large string literal - usually a TCL
+#      script of some kind.  Subsequent lines of text through the first
+#      line that begins with END_STRING are converted into a C-language
+#      string literal.
+#
+# INCLUDE path
+#
+#      The path argument is the name of a file to be inserted in place of
+#      the INCLUDE line.  The path can begin with $ROOT to signify the
+#      root of the SQLite source tree, or $HOME to signify the directory
+#      that contains the demoapp.c.in input script itself.  If the path does
+#      not begin with either $ROOT or $HOME, then it is interpreted relative
+#      to the current working directory.
+#
+#      If the INCLUDE occurs in the middle of BEGIN_STRING...END_STRING
+#      then all of the text in the input file is converted into C-language
+#      string literals.
+#
+# None of the control directives described above will nest.  Only the
+# top-level input file ("demoapp.c.in" in the example) is interpreted.
+# referenced files are copied verbatim.
+#
+if {[llength $argv]!=1} {
+  puts stderr "Usage: $argv0 TEMPLATE >OUTPUT"
+  exit 1
+}
+set infile [lindex $argv 0]
+set ROOT [file normalize [file dir $argv0]/..]
+set HOME [file normalize [file dir $infile]]
+set in [open $infile rb]
+puts [subst {/* DO NOT EDIT
+**
+** This file was generated by \"$argv0 $infile\".
+** To make changes, edit $infile then rerun the generator
+** command.
+*/}]
+set instr 0
+while {1} {
+  set line [gets $in]
+  if {[eof $in]} break
+  if {[regexp {^INCLUDE (.*)} $line all path]} {
+    regsub {^\$ROOT\y} $path $ROOT path
+    regsub {^\$HOME\y} $path $HOME path
+    set in2 [open $path rb]
+    puts "/* INCLUDE $path */"
+    if {$instr} {
+      while {1} {
+        set line [gets $in2]
+        if {[eof $in2]} break
+        set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line]
+        puts "\"$x\\n\""
+      }
+    } else {
+      puts [read $in2]
+    }
+    puts "/* END $path */"
+    close $in2
+    continue
+  }
+  if {[regexp {^BEGIN_STRING} $line]} {
+    set instr 1
+    puts "/* BEGIN_STRING */"
+    continue
+  }
+  if {[regexp {^END_STRING} $line]} {
+    set instr 0
+    puts "/* END_STRING */"
+    continue
+  }
+  if {$instr} {
+    set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line]
+    puts "\"$x\\n\""
+  } else {
+    puts $line
+  }
+}
diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in
new file mode 100644
index 0000000000..54f323a427
--- /dev/null
+++ b/tool/sqlite3_analyzer.c.in
@@ -0,0 +1,903 @@
+/*
+** Read an SQLite database file and analyze its space utilization.  Generate
+** text on standard output.
+*/
+#define TCLSH 2
+#define SQLITE_ENABLE_DBSTAT_VTAB 1
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 0
+#undef SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_OMIT_DECLTYPE 1
+#define SQLITE_OMIT_DEPRECATED 1
+#define SQLITE_OMIT_PROGRESS_CALLBACK 1
+#define SQLITE_OMIT_SHARED_CACHE 1
+#define SQLITE_DEFAULT_MEMSTATUS 0
+#define SQLITE_MAX_EXPR_DEPTH 0
+#define SQLITE_OMIT_LOAD_EXTENSION 1
+INCLUDE sqlite3.c
+INCLUDE $ROOT/src/tclsqlite.c
+
+static const char *tclsh_main_loop(void){
+return
+BEGIN_STRING
+# Run this TCL script using "testfixture" in order get a report that shows
+# how much disk space is used by a particular data to actually store data
+# versus how much space is unused.
+#
+
+if {[catch {
+
+# Argument $tname is the name of a table within the database opened by
+# database handle [db]. Return true if it is a WITHOUT ROWID table, or
+# false otherwise.
+#
+proc is_without_rowid {tname} {
+  set t [string map {' ''} $tname]
+  db eval "PRAGMA index_list = '$t'" o {
+    if {$o(origin) == "pk"} {
+      set n $o(name)
+      if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} {
+        return 1
+      }
+    }
+  }
+  return 0
+}
+
+# Read and run TCL commands from standard input.  Used to implement
+# the --tclsh option.
+#
+proc tclsh {} {
+  set line {}
+  while {![eof stdin]} {
+    if {$line!=""} {
+      puts -nonewline "> "
+    } else {
+      puts -nonewline "% "
+    }
+    flush stdout
+    append line [gets stdin]
+    if {[info complete $line]} {
+      if {[catch {uplevel #0 $line} result]} {
+        puts stderr "Error: $result"
+      } elseif {$result!=""} {
+        puts $result
+      }
+      set line {}
+    } else {
+      append line \n
+    }
+  }
+}
+
+
+# Get the name of the database to analyze
+#
+proc usage {} {
+  set argv0 [file rootname [file tail [info nameofexecutable]]]
+  puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename"
+  puts stderr {
+Analyze the SQLite3 database file specified by the "database-filename"
+argument and output a report detailing size and storage efficiency
+information for the database and its constituent tables and indexes.
+
+Options:
+
+   --pageinfo   Show how each page of the database-file is used
+
+   --stats      Output SQL text that creates a new database containing
+                statistics about the database that was analyzed
+
+   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
+
+   --version    Show the version number of SQLite
+}
+  exit 1
+}
+set file_to_analyze {}
+set flags(-pageinfo) 0
+set flags(-stats) 0
+set flags(-debug) 0
+append argv {}
+foreach arg $argv {
+  if {[regexp {^-+pageinfo$} $arg]} {
+    set flags(-pageinfo) 1
+  } elseif {[regexp {^-+stats$} $arg]} {
+    set flags(-stats) 1
+  } elseif {[regexp {^-+debug$} $arg]} {
+    set flags(-debug) 1
+  } elseif {[regexp {^-+tclsh$} $arg]} {
+    tclsh
+    exit 0
+  } elseif {[regexp {^-+version$} $arg]} {
+    sqlite3 mem :memory:
+    puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
+    mem close
+    exit 0
+  } elseif {[regexp {^-} $arg]} {
+    puts stderr "Unknown option: $arg"
+    usage
+  } elseif {$file_to_analyze!=""} {
+    usage
+  } else {
+    set file_to_analyze $arg
+  }
+}
+if {$file_to_analyze==""} usage
+set root_filename $file_to_analyze
+regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename
+if {![file exists $root_filename]} {
+  puts stderr "No such file: $root_filename"
+  exit 1
+}
+if {![file readable $root_filename]} {
+  puts stderr "File is not readable: $root_filename"
+  exit 1
+}
+set true_file_size [file size $root_filename]
+if {$true_file_size<512} {
+  puts stderr "Empty or malformed database: $root_filename"
+  exit 1
+}
+
+# Compute the total file size assuming test_multiplexor is being used.
+# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled
+#
+set extension [file extension $root_filename]
+set pattern $root_filename
+append pattern {[0-3][0-9][0-9]}
+foreach f [glob -nocomplain $pattern] {
+  incr true_file_size [file size $f]
+  set extension {}
+}
+if {[string length $extension]>=2 && [string length $extension]<=4} {
+  set pattern [file rootname $root_filename]
+  append pattern {.[0-3][0-9][0-9]}
+  foreach f [glob -nocomplain $pattern] {
+    incr true_file_size [file size $f]
+  }
+}
+
+# Open the database
+#
+if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} {
+  puts stderr "error trying to open $file_to_analyze: $msg"
+  exit 1
+}
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  db trace ::dbtrace
+}
+
+db eval {SELECT count(*) FROM sqlite_master}
+set pageSize [expr {wide([db one {PRAGMA page_size}])}]
+
+if {$flags(-pageinfo)} {
+  db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
+  db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} {
+    puts "$pageno $name $path"
+  }
+  exit 0
+}
+if {$flags(-stats)} {
+  db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
+  puts "BEGIN;"
+  puts "CREATE TABLE stats("
+  puts "  name       STRING,           /* Name of table or index */"
+  puts "  path       INTEGER,          /* Path to page from root */"
+  puts "  pageno     INTEGER,          /* Page number */"
+  puts "  pagetype   STRING,           /* 'internal', 'leaf' or 'overflow' */"
+  puts "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"
+  puts "  payload    INTEGER,          /* Bytes of payload on this page */"
+  puts "  unused     INTEGER,          /* Bytes of unused space on this page */"
+  puts "  mx_payload INTEGER,          /* Largest payload size of all cells */"
+  puts "  pgoffset   INTEGER,          /* Offset of page in file */"
+  puts "  pgsize     INTEGER           /* Size of the page */"
+  puts ");"
+  db eval {SELECT quote(name) || ',' ||
+                  quote(path) || ',' ||
+                  quote(pageno) || ',' ||
+                  quote(pagetype) || ',' ||
+                  quote(ncell) || ',' ||
+                  quote(payload) || ',' ||
+                  quote(unused) || ',' ||
+                  quote(mx_payload) || ',' ||
+                  quote(pgoffset) || ',' ||
+                  quote(pgsize) AS x FROM stat} {
+    puts "INSERT INTO stats VALUES($x);"
+  }
+  puts "COMMIT;"
+  exit 0
+}
+
+
+# In-memory database for collecting statistics. This script loops through
+# the tables and indices in the database being analyzed, adding a row for each
+# to an in-memory database (for which the schema is shown below). It then
+# queries the in-memory db to produce the space-analysis report.
+#
+sqlite3 mem :memory:
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  mem trace ::dbtrace
+}
+set tabledef {CREATE TABLE space_used(
+   name clob,        -- Name of a table or index in the database file
+   tblname clob,     -- Name of associated table
+   is_index boolean, -- TRUE if it is an index, false for a table
+   is_without_rowid boolean, -- TRUE if WITHOUT ROWID table  
+   nentry int,       -- Number of entries in the BTree
+   leaf_entries int, -- Number of leaf entries
+   depth int,        -- Depth of the b-tree
+   payload int,      -- Total amount of data stored in this table or index
+   ovfl_payload int, -- Total amount of data stored on overflow pages
+   ovfl_cnt int,     -- Number of entries that use overflow
+   mx_payload int,   -- Maximum payload size
+   int_pages int,    -- Number of interior pages used
+   leaf_pages int,   -- Number of leaf pages used
+   ovfl_pages int,   -- Number of overflow pages used
+   int_unused int,   -- Number of unused bytes on interior pages
+   leaf_unused int,  -- Number of unused bytes on primary pages
+   ovfl_unused int,  -- Number of unused bytes on overflow pages
+   gap_cnt int,      -- Number of gaps in the page layout
+   compressed_size int  -- Total bytes stored on disk
+);}
+mem eval $tabledef
+
+# Create a temporary "dbstat" virtual table.
+#
+db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
+db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat
+         ORDER BY name, path}
+db eval {DROP TABLE temp.stat}
+
+set isCompressed 0
+set compressOverhead 0
+set depth 0
+set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
+foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
+
+  set is_index [expr {$name!=$tblname}]
+  set is_without_rowid [is_without_rowid $name]
+  db eval {
+    SELECT 
+      sum(ncell) AS nentry,
+      sum((pagetype=='leaf')*ncell) AS leaf_entries,
+      sum(payload) AS payload,
+      sum((pagetype=='overflow') * payload) AS ovfl_payload,
+      sum(path LIKE '%+000000') AS ovfl_cnt,
+      max(mx_payload) AS mx_payload,
+      sum(pagetype=='internal') AS int_pages,
+      sum(pagetype=='leaf') AS leaf_pages,
+      sum(pagetype=='overflow') AS ovfl_pages,
+      sum((pagetype=='internal') * unused) AS int_unused,
+      sum((pagetype=='leaf') * unused) AS leaf_unused,
+      sum((pagetype=='overflow') * unused) AS ovfl_unused,
+      sum(pgsize) AS compressed_size,
+      max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4)
+        AS depth
+    FROM temp.dbstat WHERE name = $name
+  } break
+
+  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
+  set storage [expr {$total_pages*$pageSize}]
+  if {!$isCompressed && $storage>$compressed_size} {
+    set isCompressed 1
+    set compressOverhead 14
+  }
+
+  # Column 'gap_cnt' is set to the number of non-contiguous entries in the
+  # list of pages visited if the b-tree structure is traversed in a top-down
+  # fashion (each node visited before its child-tree is passed). Any overflow
+  # chains present are traversed from start to finish before any child-tree
+  # is.
+  #
+  set gap_cnt 0
+  set prev 0
+  db eval {
+    SELECT pageno, pagetype FROM temp.dbstat
+     WHERE name=$name
+     ORDER BY pageno
+  } {
+    if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} {
+      incr gap_cnt
+    }
+    set prev $pageno
+  }
+  mem eval {
+    INSERT INTO space_used VALUES(
+      $name,
+      $tblname,
+      $is_index,
+      $is_without_rowid,
+      $nentry,
+      $leaf_entries,
+      $depth,
+      $payload,     
+      $ovfl_payload,
+      $ovfl_cnt,   
+      $mx_payload,
+      $int_pages,
+      $leaf_pages,  
+      $ovfl_pages, 
+      $int_unused, 
+      $leaf_unused,
+      $ovfl_unused,
+      $gap_cnt,
+      $compressed_size
+    );
+  }
+}
+
+proc integerify {real} {
+  if {[string is double -strict $real]} {
+    return [expr {wide($real)}]
+  } else {
+    return 0
+  }
+}
+mem function int integerify
+
+# Quote a string for use in an SQL query. Examples:
+#
+# [quote {hello world}]   == {'hello world'}
+# [quote {hello world's}] == {'hello world''s'}
+#
+proc quote {txt} {
+  return [string map {' ''} $txt]
+}
+
+# Output a title line
+#
+proc titleline {title} {
+  if {$title==""} {
+    puts [string repeat * 79]
+  } else {
+    set len [string length $title]
+    set stars [string repeat * [expr 79-$len-5]]
+    puts "*** $title $stars"
+  }
+}
+
+# Generate a single line of output in the statistics section of the
+# report.
+#
+proc statline {title value {extra {}}} {
+  set len [string length $title]
+  set dots [string repeat . [expr 50-$len]]
+  set len [string length $value]
+  set sp2 [string range {          } $len end]
+  if {$extra ne ""} {
+    set extra " $extra"
+  }
+  puts "$title$dots $value$sp2$extra"
+}
+
+# Generate a formatted percentage value for $num/$denom
+#
+proc percent {num denom {of {}}} {
+  if {$denom==0.0} {return ""}
+  set v [expr {$num*100.0/$denom}]
+  set of {}
+  if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} {
+    return [format {%5.1f%% %s} $v $of]
+  } elseif {$v<0.1 || $v>99.9} {
+    return [format {%7.3f%% %s} $v $of]
+  } else {
+    return [format {%6.2f%% %s} $v $of]
+  }
+}
+
+proc divide {num denom} {
+  if {$denom==0} {return 0.0}
+  return [format %.2f [expr double($num)/double($denom)]]
+}
+
+# Generate a subreport that covers some subset of the database.
+# the $where clause determines which subset to analyze.
+#
+proc subreport {title where showFrag} {
+  global pageSize file_pgcnt compressOverhead
+
+  # Query the in-memory database for the sum of various statistics 
+  # for the subset of tables/indices identified by the WHERE clause in
+  # $where. Note that even if the WHERE clause matches no rows, the
+  # following query returns exactly one row (because it is an aggregate).
+  #
+  # The results of the query are stored directly by SQLite into local 
+  # variables (i.e. $nentry, $payload etc.).
+  #
+  mem eval "
+    SELECT
+      int(sum(
+        CASE WHEN (is_without_rowid OR is_index) THEN nentry 
+             ELSE leaf_entries 
+        END
+      )) AS nentry,
+      int(sum(payload)) AS payload,
+      int(sum(ovfl_payload)) AS ovfl_payload,
+      max(mx_payload) AS mx_payload,
+      int(sum(ovfl_cnt)) as ovfl_cnt,
+      int(sum(leaf_pages)) AS leaf_pages,
+      int(sum(int_pages)) AS int_pages,
+      int(sum(ovfl_pages)) AS ovfl_pages,
+      int(sum(leaf_unused)) AS leaf_unused,
+      int(sum(int_unused)) AS int_unused,
+      int(sum(ovfl_unused)) AS ovfl_unused,
+      int(sum(gap_cnt)) AS gap_cnt,
+      int(sum(compressed_size)) AS compressed_size,
+      int(max(depth)) AS depth,
+      count(*) AS cnt
+    FROM space_used WHERE $where" {} {}
+
+  # Output the sub-report title, nicely decorated with * characters.
+  #
+  puts ""
+  titleline $title
+  puts ""
+
+  # Calculate statistics and store the results in TCL variables, as follows:
+  #
+  # total_pages: Database pages consumed.
+  # total_pages_percent: Pages consumed as a percentage of the file.
+  # storage: Bytes consumed.
+  # payload_percent: Payload bytes used as a percentage of $storage.
+  # total_unused: Unused bytes on pages.
+  # avg_payload: Average payload per btree entry.
+  # avg_fanout: Average fanout for internal pages.
+  # avg_unused: Average unused bytes per btree entry.
+  # avg_meta: Average metadata overhead per entry.
+  # ovfl_cnt_percent: Percentage of btree entries that use overflow pages.
+  #
+  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
+  set total_pages_percent [percent $total_pages $file_pgcnt]
+  set storage [expr {$total_pages*$pageSize}]
+  set payload_percent [percent $payload $storage {of storage consumed}]
+  set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
+  set avg_payload [divide $payload $nentry]
+  set avg_unused [divide $total_unused $nentry]
+  set total_meta [expr {$storage - $payload - $total_unused}]
+  set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}]
+  set meta_percent [percent $total_meta $storage {of metadata}]
+  set avg_meta [divide $total_meta $nentry]
+  if {$int_pages>0} {
+    # TODO: Is this formula correct?
+    set nTab [mem eval "
+      SELECT count(*) FROM (
+          SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0
+      )
+    "]
+    set avg_fanout [mem eval "
+      SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used
+          WHERE $where
+    "]
+    set avg_fanout [format %.2f $avg_fanout]
+  }
+  set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}]
+
+  # Print out the sub-report statistics.
+  #
+  statline {Percentage of total database} $total_pages_percent
+  statline {Number of entries} $nentry
+  statline {Bytes of storage consumed} $storage
+  if {$compressed_size!=$storage} {
+    set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
+    set pct [expr {$compressed_size*100.0/$storage}]
+    set pct [format {%5.1f%%} $pct]
+    statline {Bytes used after compression} $compressed_size $pct
+  }
+  statline {Bytes of payload} $payload $payload_percent
+  statline {Bytes of metadata} $total_meta $meta_percent
+  if {$cnt==1} {statline {B-tree depth} $depth}
+  statline {Average payload per entry} $avg_payload
+  statline {Average unused bytes per entry} $avg_unused
+  statline {Average metadata per entry} $avg_meta
+  if {[info exists avg_fanout]} {
+    statline {Average fanout} $avg_fanout
+  }
+  if {$showFrag && $total_pages>1} {
+    set fragmentation [percent $gap_cnt [expr {$total_pages-1}]]
+    statline {Non-sequential pages} $gap_cnt $fragmentation
+  }
+  statline {Maximum payload per entry} $mx_payload
+  statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent
+  if {$int_pages>0} {
+    statline {Index pages used} $int_pages
+  }
+  statline {Primary pages used} $leaf_pages
+  statline {Overflow pages used} $ovfl_pages
+  statline {Total pages used} $total_pages
+  if {$int_unused>0} {
+    set int_unused_percent [
+         percent $int_unused [expr {$int_pages*$pageSize}] {of index space}]
+    statline "Unused bytes on index pages" $int_unused $int_unused_percent
+  }
+  statline "Unused bytes on primary pages" $leaf_unused [
+     percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}]
+  statline "Unused bytes on overflow pages" $ovfl_unused [
+     percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}]
+  statline "Unused bytes on all pages" $total_unused [
+               percent $total_unused $storage {of all space}]
+  return 1
+}
+
+# Calculate the overhead in pages caused by auto-vacuum. 
+#
+# This procedure calculates and returns the number of pages used by the 
+# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum,
+# then 0 is returned. The two arguments are the size of the database file in
+# pages and the page size used by the database (in bytes).
+proc autovacuum_overhead {filePages pageSize} {
+
+  # Set $autovacuum to non-zero for databases that support auto-vacuum.
+  set autovacuum [db one {PRAGMA auto_vacuum}]
+
+  # If the database is not an auto-vacuum database or the file consists
+  # of one page only then there is no overhead for auto-vacuum. Return zero.
+  if {0==$autovacuum || $filePages==1} {
+    return 0
+  }
+
+  # The number of entries on each pointer map page. The layout of the
+  # database file is one pointer-map page, followed by $ptrsPerPage other
+  # pages, followed by a pointer-map page etc. The first pointer-map page
+  # is the second page of the file overall.
+  set ptrsPerPage [expr double($pageSize/5)]
+
+  # Return the number of pointer map pages in the database.
+  return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))]
+}
+
+
+# Calculate the summary statistics for the database and store the results
+# in TCL variables. They are output below. Variables are as follows:
+#
+# pageSize:      Size of each page in bytes.
+# file_bytes:    File size in bytes.
+# file_pgcnt:    Number of pages in the file.
+# file_pgcnt2:   Number of pages in the file (calculated).
+# av_pgcnt:      Pages consumed by the auto-vacuum pointer-map.
+# av_percent:    Percentage of the file consumed by auto-vacuum pointer-map.
+# inuse_pgcnt:   Data pages in the file.
+# inuse_percent: Percentage of pages used to store data.
+# free_pgcnt:    Free pages calculated as (<total pages> - <in-use pages>)
+# free_pgcnt2:   Free pages in the file according to the file header.
+# free_percent:  Percentage of file consumed by free pages (calculated).
+# free_percent2: Percentage of file consumed by free pages (header).
+# ntable:        Number of tables in the db.
+# nindex:        Number of indices in the db.
+# nautoindex:    Number of indices created automatically.
+# nmanindex:     Number of indices created manually.
+# user_payload:  Number of bytes of payload in table btrees 
+#                (not including sqlite_master)
+# user_percent:  $user_payload as a percentage of total file size.
+
+### The following, setting $file_bytes based on the actual size of the file
+### on disk, causes this tool to choke on zipvfs databases. So set it based
+### on the return of [PRAGMA page_count] instead.
+if 0 {
+  set file_bytes  [file size $file_to_analyze]
+  set file_pgcnt  [expr {$file_bytes/$pageSize}]
+}
+set file_pgcnt  [db one {PRAGMA page_count}]
+set file_bytes  [expr {$file_pgcnt * $pageSize}]
+
+set av_pgcnt    [autovacuum_overhead $file_pgcnt $pageSize]
+set av_percent  [percent $av_pgcnt $file_pgcnt]
+
+set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used}
+set inuse_pgcnt   [expr wide([mem eval $sql])]
+set inuse_percent [percent $inuse_pgcnt $file_pgcnt]
+
+set free_pgcnt    [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}]
+set free_percent  [percent $free_pgcnt $file_pgcnt]
+set free_pgcnt2   [db one {PRAGMA freelist_count}]
+set free_percent2 [percent $free_pgcnt2 $file_pgcnt]
+
+set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}]
+
+set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}]
+set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}]
+set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'}
+set nautoindex [db eval $sql]
+set nmanindex [expr {$nindex-$nautoindex}]
+
+# set total_payload [mem eval "SELECT sum(payload) FROM space_used"]
+set user_payload [mem one {SELECT int(sum(payload)) FROM space_used
+     WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}]
+set user_percent [percent $user_payload $file_bytes]
+
+# Output the summary statistics calculated above.
+#
+puts "/** Disk-Space Utilization Report For $root_filename"
+puts ""
+statline {Page size in bytes} $pageSize
+statline {Pages in the whole file (measured)} $file_pgcnt
+statline {Pages in the whole file (calculated)} $file_pgcnt2
+statline {Pages that store data} $inuse_pgcnt $inuse_percent
+statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2
+statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent
+statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent
+statline {Number of tables in the database} $ntable
+statline {Number of indices} $nindex
+statline {Number of defined indices} $nmanindex
+statline {Number of implied indices} $nautoindex
+if {$isCompressed} {
+  statline {Size of uncompressed content in bytes} $file_bytes
+  set efficiency [percent $true_file_size $file_bytes]
+  statline {Size of compressed file on disk} $true_file_size $efficiency
+} else {
+  statline {Size of the file in bytes} $file_bytes
+}
+statline {Bytes of user payload stored} $user_payload $user_percent
+
+# Output table rankings
+#
+puts ""
+titleline "Page counts for all tables with their indices"
+puts ""
+mem eval {SELECT tblname, count(*) AS cnt, 
+              int(sum(int_pages+leaf_pages+ovfl_pages)) AS size
+          FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} {
+  statline [string toupper $tblname] $size [percent $size $file_pgcnt]
+}
+puts ""
+titleline "Page counts for all tables and indices separately"
+puts ""
+mem eval {
+  SELECT
+       upper(name) AS nm,
+       int(int_pages+leaf_pages+ovfl_pages) AS size
+    FROM space_used
+   ORDER BY size+0 DESC, name} {} {
+  statline $nm $size [percent $size $file_pgcnt]
+}
+if {$isCompressed} {
+  puts ""
+  titleline "Bytes of disk space used after compression"
+  puts ""
+  set csum 0
+  mem eval {SELECT tblname,
+                  int(sum(compressed_size)) +
+                         $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages)
+                        AS csize
+          FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} {
+    incr csum $csize
+    statline [string toupper $tblname] $csize [percent $csize $true_file_size]
+  }
+  set overhead [expr {$true_file_size - $csum}]
+  if {$overhead>0} {
+    statline {Header and free space} $overhead [percent $overhead $true_file_size]
+  }
+}
+
+# Output subreports
+#
+if {$nindex>0} {
+  subreport {All tables and indices} 1 0
+}
+subreport {All tables} {NOT is_index} 0
+if {$nindex>0} {
+  subreport {All indices} {is_index} 0
+}
+foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used
+                       ORDER BY name}] {
+  set qn [quote $tbl]
+  set name [string toupper $tbl]
+  set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}]
+  if {$n>1} {
+    set idxlist [mem eval "SELECT name FROM space_used
+                            WHERE tblname='$qn' AND is_index
+                            ORDER BY 1"]
+    subreport "Table $name and all its indices" "tblname='$qn'" 0
+    subreport "Table $name w/o any indices" "name='$qn'" 1
+    if {[llength $idxlist]>1} {
+      subreport "Indices of table $name" "tblname='$qn' AND is_index" 0
+    }
+    foreach idx $idxlist {
+      set qidx [quote $idx]
+      subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1
+    }
+  } else {
+    subreport "Table $name" "name='$qn'" 1
+  }
+}
+
+# Output instructions on what the numbers above mean.
+#
+puts ""
+titleline Definitions
+puts {
+Page size in bytes
+
+    The number of bytes in a single page of the database file.  
+    Usually 1024.
+
+Number of pages in the whole file
+}
+puts "    The number of $pageSize-byte pages that go into forming the complete
+    database"
+puts {
+Pages that store data
+
+    The number of pages that store data, either as primary B*Tree pages or
+    as overflow pages.  The number at the right is the data pages divided by
+    the total number of pages in the file.
+
+Pages on the freelist
+
+    The number of pages that are not currently in use but are reserved for
+    future use.  The percentage at the right is the number of freelist pages
+    divided by the total number of pages in the file.
+
+Pages of auto-vacuum overhead
+
+    The number of pages that store data used by the database to facilitate
+    auto-vacuum. This is zero for databases that do not support auto-vacuum.
+
+Number of tables in the database
+
+    The number of tables in the database, including the SQLITE_MASTER table
+    used to store schema information.
+
+Number of indices
+
+    The total number of indices in the database.
+
+Number of defined indices
+
+    The number of indices created using an explicit CREATE INDEX statement.
+
+Number of implied indices
+
+    The number of indices used to implement PRIMARY KEY or UNIQUE constraints
+    on tables.
+
+Size of the file in bytes
+
+    The total amount of disk space used by the entire database files.
+
+Bytes of user payload stored
+
+    The total number of bytes of user payload stored in the database. The
+    schema information in the SQLITE_MASTER table is not counted when
+    computing this number.  The percentage at the right shows the payload
+    divided by the total file size.
+
+Percentage of total database
+
+    The amount of the complete database file that is devoted to storing
+    information described by this category.
+
+Number of entries
+
+    The total number of B-Tree key/value pairs stored under this category.
+
+Bytes of storage consumed
+
+    The total amount of disk space required to store all B-Tree entries
+    under this category.  The is the total number of pages used times
+    the pages size.
+
+Bytes of payload
+
+    The amount of payload stored under this category.  Payload is the data
+    part of table entries and the key part of index entries.  The percentage
+    at the right is the bytes of payload divided by the bytes of storage 
+    consumed.
+
+Bytes of metadata
+
+    The amount of formatting and structural information stored in the
+    table or index.  Metadata includes the btree page header, the cell pointer
+    array, the size field for each cell, the left child pointer or non-leaf
+    cells, the overflow pointers for overflow cells, and the rowid value for
+    rowid table cells.  In other words, metadata is everything that is neither
+    unused space nor content.  The record header in the payload is counted as
+    content, not metadata.
+
+Average payload per entry
+
+    The average amount of payload on each entry.  This is just the bytes of
+    payload divided by the number of entries.
+
+Average unused bytes per entry
+
+    The average amount of free space remaining on all pages under this
+    category on a per-entry basis.  This is the number of unused bytes on
+    all pages divided by the number of entries.
+
+Non-sequential pages
+
+    The number of pages in the table or index that are out of sequence.
+    Many filesystems are optimized for sequential file access so a small
+    number of non-sequential pages might result in faster queries,
+    especially for larger database files that do not fit in the disk cache.
+    Note that after running VACUUM, the root page of each table or index is
+    at the beginning of the database file and all other pages are in a
+    separate part of the database file, resulting in a single non-
+    sequential page.
+
+Maximum payload per entry
+
+    The largest payload size of any entry.
+
+Entries that use overflow
+
+    The number of entries that user one or more overflow pages.
+
+Total pages used
+
+    This is the number of pages used to hold all information in the current
+    category.  This is the sum of index, primary, and overflow pages.
+
+Index pages used
+
+    This is the number of pages in a table B-tree that hold only key (rowid)
+    information and no data.
+
+Primary pages used
+
+    This is the number of B-tree pages that hold both key and data.
+
+Overflow pages used
+
+    The total number of overflow pages used for this category.
+
+Unused bytes on index pages
+
+    The total number of bytes of unused space on all index pages.  The
+    percentage at the right is the number of unused bytes divided by the
+    total number of bytes on index pages.
+
+Unused bytes on primary pages
+
+    The total number of bytes of unused space on all primary pages.  The
+    percentage at the right is the number of unused bytes divided by the
+    total number of bytes on primary pages.
+
+Unused bytes on overflow pages
+
+    The total number of bytes of unused space on all overflow pages.  The
+    percentage at the right is the number of unused bytes divided by the
+    total number of bytes on overflow pages.
+
+Unused bytes on all pages
+
+    The total number of bytes of unused space on all primary and overflow 
+    pages.  The percentage at the right is the number of unused bytes 
+    divided by the total number of bytes.
+}
+
+# Output a dump of the in-memory database. This can be used for more
+# complex offline analysis.
+#
+titleline {}
+puts "The entire text of this report can be sourced into any SQL database"
+puts "engine for further analysis.  All of the text above is an SQL comment."
+puts "The data used to generate this report follows:"
+puts "*/"
+puts "BEGIN;"
+puts $tabledef
+unset -nocomplain x
+mem eval {SELECT * FROM space_used} x {
+  puts -nonewline "INSERT INTO space_used VALUES"
+  set sep (
+  foreach col $x(*) {
+    set v $x($col)
+    if {$v=="" || ![string is double $v]} {set v '[quote $v]'}
+    puts -nonewline $sep$v
+    set sep ,
+  }
+  puts ");"
+}
+puts "COMMIT;"
+
+} err]} {
+  puts "ERROR: $err"
+  puts $errorInfo
+  exit 1
+}
+END_STRING
+;
+}
diff --git a/tool/tostr.tcl b/tool/tostr.tcl
deleted file mode 100644
index cb06ee947f..0000000000
--- a/tool/tostr.tcl
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/tcl
-#
-# Convert input text into a C string
-#
-set in [open [lindex $argv 0] rb]
-while {![eof $in]} {
-  set line [gets $in]
-  if {[eof $in]} break;
-  set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line]
-  puts "\"$x\\n\""
-}
-close $in

From 96a206fa1055e1f6fa3d0bf8a221bdf18bc7195d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 13 Oct 2017 20:14:06 +0000
Subject: [PATCH 189/270] Improved ability to generate stand-alone program
 using TCL and SQLite by compiling with -DTCLSH_INIT_PROC=name to cause the
 TCL interpreter to be initialized using procedure name().  Both
 sqlite3_analyzer and testfixture are now built this way.

FossilOrigin-Name: d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4
---
 Makefile.in                |   5 +-
 Makefile.msc               |   4 +-
 main.mk                    |   9 +-
 manifest                   |  22 +-
 manifest.uuid              |   2 +-
 src/tclsqlite.c            | 138 +++---
 src/test_tclsh.c           | 194 ++++----
 tool/sqlite3_analyzer.c.in | 886 +------------------------------------
 8 files changed, 187 insertions(+), 1073 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index e2934d80b1..5f6be57c37 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -947,7 +947,7 @@ tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
-	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
+	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
 
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
@@ -1113,7 +1113,8 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
diff --git a/Makefile.msc b/Makefile.msc
index bfd368c558..ce867ba3f3 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1934,7 +1934,7 @@ tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
-	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
+	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
 	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
@@ -2105,7 +2105,7 @@ sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 # necessary because the test fixture requires non-API symbols which are
 # hidden when the library is built via the amalgamation).
 #
-TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
diff --git a/main.mk b/main.mk
index 18d7cd3459..c7711fb6e9 100644
--- a/main.mk
+++ b/main.mk
@@ -777,7 +777,7 @@ sqlite3rbu.o:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 # Rules for building test programs and for running tests
 #
 tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
-	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
+	$(TCCX) $(TCL_FLAGS) -DTCLSH -o tclsqlite3 \
 		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
 
 sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl
@@ -798,21 +798,22 @@ TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
+TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
 
 testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
-	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
 		$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c                \
 		-o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB)
 
 amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c  \
 				$(TOP)/ext/session/test_session.c
-	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
 		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c                  \
 		$(TOP)/ext/session/test_session.c                            \
 		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
 
 fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
-	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
 	-DSQLITE_ENABLE_FTS3=1                                               \
 		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
 		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
diff --git a/manifest b/manifest
index 669c1b284a..2599e89765 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Add\sthe\stcl/mkccode.tcl\sscript\sused\sto\sconstruct\sa\ssingle\sC-language\ssource\nfiel\sfor\sprograms\sthat\scombine\sC-code,\sSQLite,\sand\sTCL.\s\sUse\sthis\sscript\sto\nconstruct\sthe\ssqlite3_analyzer\sprogram.
-D 2017-10-13T18:58:55.897
-F Makefile.in a99044d778e2a83ba6cf78b88937d3d667cee425dca5915d73eeba1d56b98082
+C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway.
+D 2017-10-13T20:14:06.865
+F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b864c4f906031a37e425466d8214df2da2f01703956dad5290156f03548d7277
+F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -384,7 +384,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a1948e0509f143fe4395a449cf61039bb6c7db3a93ae6af65092a72556718026
+F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -470,7 +470,7 @@ F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53b
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 7ffde8e2cd556ed74b5d40ae0ee924aa974ee3b3dacd5114c5c98feb9f651ac4
+F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
 F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
@@ -514,7 +514,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
 F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef
 F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
-F src/test_tclsh.c 2a1ff3eac1ac3ee070a80209384d6cb353d4ad4a549400e3cd29445c672b2b87
+F src/test_tclsh.c 74fcfb7f3b0ff1f871e62263dd84ffba46a8e9d477439115e0fb2035e4bf69e1
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
@@ -1632,7 +1632,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
-F tool/sqlite3_analyzer.c.in 3d53f06e04619f43b7aebc01cc318ef215bfd09ea1947fe9e485bed6aa6cb4b9
+F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a9c4bc88fcf985a0bea14ed5381239cfb697886287998da04a10230b6858ab5d
-R 78f34b33921be1c7a7c8587f43b33b81
+P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82
+R df005e0c0c347d89af378f70d82f057b
 U drh
-Z 180ea09ee34fefc9e97e7461c13719ef
+Z 8b50e8b5594af446543ab2fc55fee9a2
diff --git a/manifest.uuid b/manifest.uuid
index fabaeec3c0..d905d370b6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82
\ No newline at end of file
+d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 0009eab695..eed86eee34 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -14,17 +14,19 @@
 **
 ** Compile-time options:
 **
-**  -DTCLSH=1             Add a "main()" routine that works as a tclsh.
+**  -DTCLSH         Add a "main()" routine that works as a tclsh.
 **
-**  -DSQLITE_TCLMD5       When used in conjuction with -DTCLSH=1, add
-**                        four new commands to the TCL interpreter for
-**                        generating MD5 checksums:  md5, md5file,
-**                        md5-10x8, and md5file-10x8.
+**  -DTCLSH_INIT_PROC=name
 **
-**  -DSQLITE_TEST         When used in conjuction with -DTCLSH=1, add
-**                        hundreds of new commands used for testing
-**                        SQLite.  This option implies -DSQLITE_TCLMD5.
+**                  Invoke name(interp) to initialize the Tcl interpreter.
+**                  If name(interp) returns a non-NULL string, then run
+**                  that string as a Tcl script to launch the application.
+**                  If name(interp) returns NULL, then run the regular
+**                  tclsh-emulator code.
 */
+#ifdef TCLSH_INIT_PROC
+# define TCLSH 1
+#endif
 
 /*
 ** If requested, include the SQLite compiler options file for MSVC.
@@ -3582,56 +3584,55 @@ int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
 #endif
 
 /*
-** If the TCLSH macro is defined to be either 1 or 2, then a main()
-** routine is inserted that starts up a Tcl interpreter.  When TCLSH==1
-** the interpreter works like an ordinary tclsh.  When TCLSH==2 then the
-** startup script is supplied by an routine named "tclsh_main_loop()"
-** that must be linked separately.  The TCLSH==2 technique is used to
-** generate stand-alone executables based on TCL, such as 
-** sqlite3_analyzer.exe.
+** If the TCLSH macro is defined, add code to make a stand-alone program.
 */
-#ifdef TCLSH
+#if defined(TCLSH)
 
-/*
-** If the macro TCLSH is one, then put in code this for the
-** "main" routine that will initialize Tcl and take input from
-** standard input, or if a file is named on the command line
-** the TCL interpreter reads and evaluates that file.
+/* This is the main routine for an ordinary TCL shell.  If there are
+** are arguments, run the first argument as a script.  Otherwise,
+** read TCL commands from standard input
 */
-#if TCLSH==1
 static const char *tclsh_main_loop(void){
   static const char zMainloop[] =
-    "set line {}\n"
-    "while {![eof stdin]} {\n"
-      "if {$line!=\"\"} {\n"
-        "puts -nonewline \"> \"\n"
-      "} else {\n"
-        "puts -nonewline \"% \"\n"
-      "}\n"
-      "flush stdout\n"
-      "append line [gets stdin]\n"
-      "if {[info complete $line]} {\n"
-        "if {[catch {uplevel #0 $line} result]} {\n"
-          "puts stderr \"Error: $result\"\n"
-        "} elseif {$result!=\"\"} {\n"
-          "puts $result\n"
+    "if {[llength $argv]>=1} {\n"
+      "set argv0 [lindex $argv 0]\n"
+      "set argv [lrange $argv 1 end]\n"
+      "source $argv0\n"
+    "} else {\n"
+      "set line {}\n"
+      "while {![eof stdin]} {\n"
+        "if {$line!=\"\"} {\n"
+          "puts -nonewline \"> \"\n"
+        "} else {\n"
+          "puts -nonewline \"% \"\n"
+        "}\n"
+        "flush stdout\n"
+        "append line [gets stdin]\n"
+        "if {[info complete $line]} {\n"
+          "if {[catch {uplevel #0 $line} result]} {\n"
+            "puts stderr \"Error: $result\"\n"
+          "} elseif {$result!=\"\"} {\n"
+            "puts $result\n"
+          "}\n"
+          "set line {}\n"
+        "} else {\n"
+          "append line \\n\n"
         "}\n"
-        "set line {}\n"
-      "} else {\n"
-        "append line \\n\n"
       "}\n"
     "}\n"
   ;
   return zMainloop;
 }
-#endif
-#if TCLSH==2
-static const char *tclsh_main_loop(void);
-#endif
 
 #define TCLSH_MAIN main   /* Needed to fake out mktclapp */
 int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_Interp *interp;
+  int i;
+  const char *zScript = 0;
+  char zArgc[32];
+#if defined(TCLSH_INIT_PROC)
+  extern const char *TCLSH_INIT_PROC(Tcl_Interp*);
+#endif
 
 #if !defined(_WIN32_WCE)
   if( getenv("BREAK") ){
@@ -3650,42 +3651,27 @@ int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_FindExecutable(argv[0]);
   Tcl_SetSystemEncoding(NULL, "utf-8");
   interp = Tcl_CreateInterp();
-
-#if TCLSH==2
-  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
-#endif
-
-  /* Add extensions */
-#if !defined(SQLITE_TEST)
-  /* Normally we only initialize the TCL extension */
   Sqlite3_Init(interp);
-#else
-  /* For testing, do lots of extra initialization */
-  {
-    extern void sqlite3InitTclTestLogic(Tcl_Interp*);
-    sqlite3InitTclTestLogic(interp);
+
+  sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-1);
+  Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY);
+  Tcl_SetVar(interp,"argv0",argv[0],TCL_GLOBAL_ONLY);
+  Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY);
+  for(i=1; i<argc; i++){
+    Tcl_SetVar(interp, "argv", argv[i],
+        TCL_GLOBAL_ONLY | TCL_LIST_ELEMENT | TCL_APPEND_VALUE);
   }
-#endif /* SQLITE_TEST */
-  if( argc>=2 ){
-    int i;
-    char zArgc[32];
-    sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-(3-TCLSH));
-    Tcl_SetVar(interp,"argc", zArgc, TCL_GLOBAL_ONLY);
-    Tcl_SetVar(interp,"argv0",argv[1],TCL_GLOBAL_ONLY);
-    Tcl_SetVar(interp,"argv", "", TCL_GLOBAL_ONLY);
-    for(i=3-TCLSH; i<argc; i++){
-      Tcl_SetVar(interp, "argv", argv[i],
-          TCL_GLOBAL_ONLY | TCL_LIST_ELEMENT | TCL_APPEND_VALUE);
-    }
-    if( TCLSH==1 && Tcl_EvalFile(interp, argv[1])!=TCL_OK ){
-      const char *zInfo = Tcl_GetVar(interp, "errorInfo", TCL_GLOBAL_ONLY);
-      if( zInfo==0 ) zInfo = Tcl_GetStringResult(interp);
-      fprintf(stderr,"%s: %s\n", *argv, zInfo);
-      return 1;
-    }
+#if defined(TCLSH_INIT_PROC)
+  zScript = TCLSH_INIT_PROC(interp);
+#endif
+  if( zScript==0 ){
+    zScript = tclsh_main_loop();
   }
-  if( TCLSH==2 || argc<=1 ){
-    Tcl_GlobalEval(interp, tclsh_main_loop());
+  if( Tcl_GlobalEval(interp, zScript)!=TCL_OK ){
+    const char *zInfo = Tcl_GetVar(interp, "errorInfo", TCL_GLOBAL_ONLY);
+    if( zInfo==0 ) zInfo = Tcl_GetStringResult(interp);
+    fprintf(stderr,"%s: %s\n", *argv, zInfo);
+    return 1;
   }
   return 0;
 }
diff --git a/src/test_tclsh.c b/src/test_tclsh.c
index 213f5b1fe1..976f7cb248 100644
--- a/src/test_tclsh.c
+++ b/src/test_tclsh.c
@@ -55,7 +55,57 @@ static int SQLITE_TCLAPI load_testfixture_extensions(
 **   * If SQLITE_TEST is set, the various test interfaces used by the Tcl
 **     test suite.
 */
-void sqlite3InitTclTestLogic(Tcl_Interp *interp){
+const char *sqlite3TestInit(Tcl_Interp *interp){
+  extern int Sqlite3_Init(Tcl_Interp*);
+  extern int Sqliteconfig_Init(Tcl_Interp*);
+  extern int Sqlitetest1_Init(Tcl_Interp*);
+  extern int Sqlitetest2_Init(Tcl_Interp*);
+  extern int Sqlitetest3_Init(Tcl_Interp*);
+  extern int Sqlitetest4_Init(Tcl_Interp*);
+  extern int Sqlitetest5_Init(Tcl_Interp*);
+  extern int Sqlitetest6_Init(Tcl_Interp*);
+  extern int Sqlitetest7_Init(Tcl_Interp*);
+  extern int Sqlitetest8_Init(Tcl_Interp*);
+  extern int Sqlitetest9_Init(Tcl_Interp*);
+  extern int Sqlitetestasync_Init(Tcl_Interp*);
+  extern int Sqlitetest_autoext_Init(Tcl_Interp*);
+  extern int Sqlitetest_blob_Init(Tcl_Interp*);
+  extern int Sqlitetest_demovfs_Init(Tcl_Interp *);
+  extern int Sqlitetest_func_Init(Tcl_Interp*);
+  extern int Sqlitetest_hexio_Init(Tcl_Interp*);
+  extern int Sqlitetest_init_Init(Tcl_Interp*);
+  extern int Sqlitetest_malloc_Init(Tcl_Interp*);
+  extern int Sqlitetest_mutex_Init(Tcl_Interp*);
+  extern int Sqlitetestschema_Init(Tcl_Interp*);
+  extern int Sqlitetestsse_Init(Tcl_Interp*);
+  extern int Sqlitetesttclvar_Init(Tcl_Interp*);
+  extern int Sqlitetestfs_Init(Tcl_Interp*);
+  extern int SqlitetestThread_Init(Tcl_Interp*);
+  extern int SqlitetestOnefile_Init();
+  extern int SqlitetestOsinst_Init(Tcl_Interp*);
+  extern int Sqlitetestbackup_Init(Tcl_Interp*);
+  extern int Sqlitetestintarray_Init(Tcl_Interp*);
+  extern int Sqlitetestvfs_Init(Tcl_Interp *);
+  extern int Sqlitetestrtree_Init(Tcl_Interp*);
+  extern int Sqlitequota_Init(Tcl_Interp*);
+  extern int Sqlitemultiplex_Init(Tcl_Interp*);
+  extern int SqliteSuperlock_Init(Tcl_Interp*);
+  extern int SqlitetestSyscall_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+  extern int TestSession_Init(Tcl_Interp*);
+#endif
+  extern int Md5_Init(Tcl_Interp*);
+  extern int Fts5tcl_Init(Tcl_Interp *);
+  extern int SqliteRbu_Init(Tcl_Interp*);
+  extern int Sqlitetesttcl_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
+  extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
+#endif
+#ifdef SQLITE_ENABLE_ZIPVFS
+  extern int Zipvfs_Init(Tcl_Interp*);
+#endif
+  Tcl_CmdInfo cmdInfo;
+
   /* Since the primary use case for this binary is testing of SQLite,
   ** be sure to generate core files if we crash */
 #if defined(unix)
@@ -66,109 +116,61 @@ void sqlite3InitTclTestLogic(Tcl_Interp *interp){
   }
 #endif /* unix */
 
-  {
-    extern int Sqlite3_Init(Tcl_Interp*);
-    extern int Sqliteconfig_Init(Tcl_Interp*);
-    extern int Sqlitetest1_Init(Tcl_Interp*);
-    extern int Sqlitetest2_Init(Tcl_Interp*);
-    extern int Sqlitetest3_Init(Tcl_Interp*);
-    extern int Sqlitetest4_Init(Tcl_Interp*);
-    extern int Sqlitetest5_Init(Tcl_Interp*);
-    extern int Sqlitetest6_Init(Tcl_Interp*);
-    extern int Sqlitetest7_Init(Tcl_Interp*);
-    extern int Sqlitetest8_Init(Tcl_Interp*);
-    extern int Sqlitetest9_Init(Tcl_Interp*);
-    extern int Sqlitetestasync_Init(Tcl_Interp*);
-    extern int Sqlitetest_autoext_Init(Tcl_Interp*);
-    extern int Sqlitetest_blob_Init(Tcl_Interp*);
-    extern int Sqlitetest_demovfs_Init(Tcl_Interp *);
-    extern int Sqlitetest_func_Init(Tcl_Interp*);
-    extern int Sqlitetest_hexio_Init(Tcl_Interp*);
-    extern int Sqlitetest_init_Init(Tcl_Interp*);
-    extern int Sqlitetest_malloc_Init(Tcl_Interp*);
-    extern int Sqlitetest_mutex_Init(Tcl_Interp*);
-    extern int Sqlitetestschema_Init(Tcl_Interp*);
-    extern int Sqlitetestsse_Init(Tcl_Interp*);
-    extern int Sqlitetesttclvar_Init(Tcl_Interp*);
-    extern int Sqlitetestfs_Init(Tcl_Interp*);
-    extern int SqlitetestThread_Init(Tcl_Interp*);
-    extern int SqlitetestOnefile_Init();
-    extern int SqlitetestOsinst_Init(Tcl_Interp*);
-    extern int Sqlitetestbackup_Init(Tcl_Interp*);
-    extern int Sqlitetestintarray_Init(Tcl_Interp*);
-    extern int Sqlitetestvfs_Init(Tcl_Interp *);
-    extern int Sqlitetestrtree_Init(Tcl_Interp*);
-    extern int Sqlitequota_Init(Tcl_Interp*);
-    extern int Sqlitemultiplex_Init(Tcl_Interp*);
-    extern int SqliteSuperlock_Init(Tcl_Interp*);
-    extern int SqlitetestSyscall_Init(Tcl_Interp*);
-#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
-    extern int TestSession_Init(Tcl_Interp*);
-#endif
-    extern int Md5_Init(Tcl_Interp*);
-    extern int Fts5tcl_Init(Tcl_Interp *);
-    extern int SqliteRbu_Init(Tcl_Interp*);
-    extern int Sqlitetesttcl_Init(Tcl_Interp*);
-#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
-    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
-#endif
-#ifdef SQLITE_ENABLE_ZIPVFS
-    extern int Zipvfs_Init(Tcl_Interp*);
-#endif
-
+  if( Tcl_GetCommandInfo(interp, "sqlite3", &cmdInfo)==0 ){
     Sqlite3_Init(interp);
+  }
 #ifdef SQLITE_ENABLE_ZIPVFS
-    Zipvfs_Init(interp);
+  Zipvfs_Init(interp);
 #endif
-    Md5_Init(interp);
-    Sqliteconfig_Init(interp);
-    Sqlitetest1_Init(interp);
-    Sqlitetest2_Init(interp);
-    Sqlitetest3_Init(interp);
-    Sqlitetest4_Init(interp);
-    Sqlitetest5_Init(interp);
-    Sqlitetest6_Init(interp);
-    Sqlitetest7_Init(interp);
-    Sqlitetest8_Init(interp);
-    Sqlitetest9_Init(interp);
-    Sqlitetestasync_Init(interp);
-    Sqlitetest_autoext_Init(interp);
-    Sqlitetest_blob_Init(interp);
-    Sqlitetest_demovfs_Init(interp);
-    Sqlitetest_func_Init(interp);
-    Sqlitetest_hexio_Init(interp);
-    Sqlitetest_init_Init(interp);
-    Sqlitetest_malloc_Init(interp);
-    Sqlitetest_mutex_Init(interp);
-    Sqlitetestschema_Init(interp);
-    Sqlitetesttclvar_Init(interp);
-    Sqlitetestfs_Init(interp);
-    SqlitetestThread_Init(interp);
-    SqlitetestOnefile_Init();
-    SqlitetestOsinst_Init(interp);
-    Sqlitetestbackup_Init(interp);
-    Sqlitetestintarray_Init(interp);
-    Sqlitetestvfs_Init(interp);
-    Sqlitetestrtree_Init(interp);
-    Sqlitequota_Init(interp);
-    Sqlitemultiplex_Init(interp);
-    SqliteSuperlock_Init(interp);
-    SqlitetestSyscall_Init(interp);
+  Md5_Init(interp);
+  Sqliteconfig_Init(interp);
+  Sqlitetest1_Init(interp);
+  Sqlitetest2_Init(interp);
+  Sqlitetest3_Init(interp);
+  Sqlitetest4_Init(interp);
+  Sqlitetest5_Init(interp);
+  Sqlitetest6_Init(interp);
+  Sqlitetest7_Init(interp);
+  Sqlitetest8_Init(interp);
+  Sqlitetest9_Init(interp);
+  Sqlitetestasync_Init(interp);
+  Sqlitetest_autoext_Init(interp);
+  Sqlitetest_blob_Init(interp);
+  Sqlitetest_demovfs_Init(interp);
+  Sqlitetest_func_Init(interp);
+  Sqlitetest_hexio_Init(interp);
+  Sqlitetest_init_Init(interp);
+  Sqlitetest_malloc_Init(interp);
+  Sqlitetest_mutex_Init(interp);
+  Sqlitetestschema_Init(interp);
+  Sqlitetesttclvar_Init(interp);
+  Sqlitetestfs_Init(interp);
+  SqlitetestThread_Init(interp);
+  SqlitetestOnefile_Init();
+  SqlitetestOsinst_Init(interp);
+  Sqlitetestbackup_Init(interp);
+  Sqlitetestintarray_Init(interp);
+  Sqlitetestvfs_Init(interp);
+  Sqlitetestrtree_Init(interp);
+  Sqlitequota_Init(interp);
+  Sqlitemultiplex_Init(interp);
+  SqliteSuperlock_Init(interp);
+  SqlitetestSyscall_Init(interp);
 #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
-    TestSession_Init(interp);
+  TestSession_Init(interp);
 #endif
-    Fts5tcl_Init(interp);
-    SqliteRbu_Init(interp);
-    Sqlitetesttcl_Init(interp);
+  Fts5tcl_Init(interp);
+  SqliteRbu_Init(interp);
+  Sqlitetesttcl_Init(interp);
 
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
-    Sqlitetestfts3_Init(interp);
+  Sqlitetestfts3_Init(interp);
 #endif
 
-    Tcl_CreateObjCommand(
-        interp, "load_testfixture_extensions", load_testfixture_extensions,0,0
-    );
-  }
+  Tcl_CreateObjCommand(
+      interp, "load_testfixture_extensions", load_testfixture_extensions,0,0
+  );
+  return 0;
 }
 
 /* tclcmd:   load_testfixture_extensions
@@ -191,6 +193,6 @@ static int SQLITE_TCLAPI load_testfixture_extensions(
     return TCL_ERROR;
   }
 
-  sqlite3InitTclTestLogic(slave);
+  (void)sqlite3TestInit(slave);
   return TCL_OK;
 }
diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in
index 54f323a427..547b22ffe1 100644
--- a/tool/sqlite3_analyzer.c.in
+++ b/tool/sqlite3_analyzer.c.in
@@ -2,7 +2,7 @@
 ** Read an SQLite database file and analyze its space utilization.  Generate
 ** text on standard output.
 */
-#define TCLSH 2
+#define TCLSH_INIT_PROC sqlite3_analyzer_init_proc
 #define SQLITE_ENABLE_DBSTAT_VTAB 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0
@@ -17,887 +17,11 @@
 INCLUDE sqlite3.c
 INCLUDE $ROOT/src/tclsqlite.c
 
-static const char *tclsh_main_loop(void){
-return
+const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){
+  (void)interp;
+  return
 BEGIN_STRING
-# Run this TCL script using "testfixture" in order get a report that shows
-# how much disk space is used by a particular data to actually store data
-# versus how much space is unused.
-#
-
-if {[catch {
-
-# Argument $tname is the name of a table within the database opened by
-# database handle [db]. Return true if it is a WITHOUT ROWID table, or
-# false otherwise.
-#
-proc is_without_rowid {tname} {
-  set t [string map {' ''} $tname]
-  db eval "PRAGMA index_list = '$t'" o {
-    if {$o(origin) == "pk"} {
-      set n $o(name)
-      if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} {
-        return 1
-      }
-    }
-  }
-  return 0
-}
-
-# Read and run TCL commands from standard input.  Used to implement
-# the --tclsh option.
-#
-proc tclsh {} {
-  set line {}
-  while {![eof stdin]} {
-    if {$line!=""} {
-      puts -nonewline "> "
-    } else {
-      puts -nonewline "% "
-    }
-    flush stdout
-    append line [gets stdin]
-    if {[info complete $line]} {
-      if {[catch {uplevel #0 $line} result]} {
-        puts stderr "Error: $result"
-      } elseif {$result!=""} {
-        puts $result
-      }
-      set line {}
-    } else {
-      append line \n
-    }
-  }
-}
-
-
-# Get the name of the database to analyze
-#
-proc usage {} {
-  set argv0 [file rootname [file tail [info nameofexecutable]]]
-  puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename"
-  puts stderr {
-Analyze the SQLite3 database file specified by the "database-filename"
-argument and output a report detailing size and storage efficiency
-information for the database and its constituent tables and indexes.
-
-Options:
-
-   --pageinfo   Show how each page of the database-file is used
-
-   --stats      Output SQL text that creates a new database containing
-                statistics about the database that was analyzed
-
-   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
-
-   --version    Show the version number of SQLite
-}
-  exit 1
-}
-set file_to_analyze {}
-set flags(-pageinfo) 0
-set flags(-stats) 0
-set flags(-debug) 0
-append argv {}
-foreach arg $argv {
-  if {[regexp {^-+pageinfo$} $arg]} {
-    set flags(-pageinfo) 1
-  } elseif {[regexp {^-+stats$} $arg]} {
-    set flags(-stats) 1
-  } elseif {[regexp {^-+debug$} $arg]} {
-    set flags(-debug) 1
-  } elseif {[regexp {^-+tclsh$} $arg]} {
-    tclsh
-    exit 0
-  } elseif {[regexp {^-+version$} $arg]} {
-    sqlite3 mem :memory:
-    puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
-    mem close
-    exit 0
-  } elseif {[regexp {^-} $arg]} {
-    puts stderr "Unknown option: $arg"
-    usage
-  } elseif {$file_to_analyze!=""} {
-    usage
-  } else {
-    set file_to_analyze $arg
-  }
-}
-if {$file_to_analyze==""} usage
-set root_filename $file_to_analyze
-regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename
-if {![file exists $root_filename]} {
-  puts stderr "No such file: $root_filename"
-  exit 1
-}
-if {![file readable $root_filename]} {
-  puts stderr "File is not readable: $root_filename"
-  exit 1
-}
-set true_file_size [file size $root_filename]
-if {$true_file_size<512} {
-  puts stderr "Empty or malformed database: $root_filename"
-  exit 1
-}
-
-# Compute the total file size assuming test_multiplexor is being used.
-# Assume that SQLITE_ENABLE_8_3_NAMES might be enabled
-#
-set extension [file extension $root_filename]
-set pattern $root_filename
-append pattern {[0-3][0-9][0-9]}
-foreach f [glob -nocomplain $pattern] {
-  incr true_file_size [file size $f]
-  set extension {}
-}
-if {[string length $extension]>=2 && [string length $extension]<=4} {
-  set pattern [file rootname $root_filename]
-  append pattern {.[0-3][0-9][0-9]}
-  foreach f [glob -nocomplain $pattern] {
-    incr true_file_size [file size $f]
-  }
-}
-
-# Open the database
-#
-if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} {
-  puts stderr "error trying to open $file_to_analyze: $msg"
-  exit 1
-}
-if {$flags(-debug)} {
-  proc dbtrace {txt} {puts $txt; flush stdout;}
-  db trace ::dbtrace
-}
-
-db eval {SELECT count(*) FROM sqlite_master}
-set pageSize [expr {wide([db one {PRAGMA page_size}])}]
-
-if {$flags(-pageinfo)} {
-  db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
-  db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} {
-    puts "$pageno $name $path"
-  }
-  exit 0
-}
-if {$flags(-stats)} {
-  db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
-  puts "BEGIN;"
-  puts "CREATE TABLE stats("
-  puts "  name       STRING,           /* Name of table or index */"
-  puts "  path       INTEGER,          /* Path to page from root */"
-  puts "  pageno     INTEGER,          /* Page number */"
-  puts "  pagetype   STRING,           /* 'internal', 'leaf' or 'overflow' */"
-  puts "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"
-  puts "  payload    INTEGER,          /* Bytes of payload on this page */"
-  puts "  unused     INTEGER,          /* Bytes of unused space on this page */"
-  puts "  mx_payload INTEGER,          /* Largest payload size of all cells */"
-  puts "  pgoffset   INTEGER,          /* Offset of page in file */"
-  puts "  pgsize     INTEGER           /* Size of the page */"
-  puts ");"
-  db eval {SELECT quote(name) || ',' ||
-                  quote(path) || ',' ||
-                  quote(pageno) || ',' ||
-                  quote(pagetype) || ',' ||
-                  quote(ncell) || ',' ||
-                  quote(payload) || ',' ||
-                  quote(unused) || ',' ||
-                  quote(mx_payload) || ',' ||
-                  quote(pgoffset) || ',' ||
-                  quote(pgsize) AS x FROM stat} {
-    puts "INSERT INTO stats VALUES($x);"
-  }
-  puts "COMMIT;"
-  exit 0
-}
-
-
-# In-memory database for collecting statistics. This script loops through
-# the tables and indices in the database being analyzed, adding a row for each
-# to an in-memory database (for which the schema is shown below). It then
-# queries the in-memory db to produce the space-analysis report.
-#
-sqlite3 mem :memory:
-if {$flags(-debug)} {
-  proc dbtrace {txt} {puts $txt; flush stdout;}
-  mem trace ::dbtrace
-}
-set tabledef {CREATE TABLE space_used(
-   name clob,        -- Name of a table or index in the database file
-   tblname clob,     -- Name of associated table
-   is_index boolean, -- TRUE if it is an index, false for a table
-   is_without_rowid boolean, -- TRUE if WITHOUT ROWID table  
-   nentry int,       -- Number of entries in the BTree
-   leaf_entries int, -- Number of leaf entries
-   depth int,        -- Depth of the b-tree
-   payload int,      -- Total amount of data stored in this table or index
-   ovfl_payload int, -- Total amount of data stored on overflow pages
-   ovfl_cnt int,     -- Number of entries that use overflow
-   mx_payload int,   -- Maximum payload size
-   int_pages int,    -- Number of interior pages used
-   leaf_pages int,   -- Number of leaf pages used
-   ovfl_pages int,   -- Number of overflow pages used
-   int_unused int,   -- Number of unused bytes on interior pages
-   leaf_unused int,  -- Number of unused bytes on primary pages
-   ovfl_unused int,  -- Number of unused bytes on overflow pages
-   gap_cnt int,      -- Number of gaps in the page layout
-   compressed_size int  -- Total bytes stored on disk
-);}
-mem eval $tabledef
-
-# Create a temporary "dbstat" virtual table.
-#
-db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
-db eval {CREATE TEMP TABLE dbstat AS SELECT * FROM temp.stat
-         ORDER BY name, path}
-db eval {DROP TABLE temp.stat}
-
-set isCompressed 0
-set compressOverhead 0
-set depth 0
-set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
-foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
-
-  set is_index [expr {$name!=$tblname}]
-  set is_without_rowid [is_without_rowid $name]
-  db eval {
-    SELECT 
-      sum(ncell) AS nentry,
-      sum((pagetype=='leaf')*ncell) AS leaf_entries,
-      sum(payload) AS payload,
-      sum((pagetype=='overflow') * payload) AS ovfl_payload,
-      sum(path LIKE '%+000000') AS ovfl_cnt,
-      max(mx_payload) AS mx_payload,
-      sum(pagetype=='internal') AS int_pages,
-      sum(pagetype=='leaf') AS leaf_pages,
-      sum(pagetype=='overflow') AS ovfl_pages,
-      sum((pagetype=='internal') * unused) AS int_unused,
-      sum((pagetype=='leaf') * unused) AS leaf_unused,
-      sum((pagetype=='overflow') * unused) AS ovfl_unused,
-      sum(pgsize) AS compressed_size,
-      max((length(CASE WHEN path LIKE '%+%' THEN '' ELSE path END)+3)/4)
-        AS depth
-    FROM temp.dbstat WHERE name = $name
-  } break
-
-  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
-  set storage [expr {$total_pages*$pageSize}]
-  if {!$isCompressed && $storage>$compressed_size} {
-    set isCompressed 1
-    set compressOverhead 14
-  }
-
-  # Column 'gap_cnt' is set to the number of non-contiguous entries in the
-  # list of pages visited if the b-tree structure is traversed in a top-down
-  # fashion (each node visited before its child-tree is passed). Any overflow
-  # chains present are traversed from start to finish before any child-tree
-  # is.
-  #
-  set gap_cnt 0
-  set prev 0
-  db eval {
-    SELECT pageno, pagetype FROM temp.dbstat
-     WHERE name=$name
-     ORDER BY pageno
-  } {
-    if {$prev>0 && $pagetype=="leaf" && $pageno!=$prev+1} {
-      incr gap_cnt
-    }
-    set prev $pageno
-  }
-  mem eval {
-    INSERT INTO space_used VALUES(
-      $name,
-      $tblname,
-      $is_index,
-      $is_without_rowid,
-      $nentry,
-      $leaf_entries,
-      $depth,
-      $payload,     
-      $ovfl_payload,
-      $ovfl_cnt,   
-      $mx_payload,
-      $int_pages,
-      $leaf_pages,  
-      $ovfl_pages, 
-      $int_unused, 
-      $leaf_unused,
-      $ovfl_unused,
-      $gap_cnt,
-      $compressed_size
-    );
-  }
-}
-
-proc integerify {real} {
-  if {[string is double -strict $real]} {
-    return [expr {wide($real)}]
-  } else {
-    return 0
-  }
-}
-mem function int integerify
-
-# Quote a string for use in an SQL query. Examples:
-#
-# [quote {hello world}]   == {'hello world'}
-# [quote {hello world's}] == {'hello world''s'}
-#
-proc quote {txt} {
-  return [string map {' ''} $txt]
-}
-
-# Output a title line
-#
-proc titleline {title} {
-  if {$title==""} {
-    puts [string repeat * 79]
-  } else {
-    set len [string length $title]
-    set stars [string repeat * [expr 79-$len-5]]
-    puts "*** $title $stars"
-  }
-}
-
-# Generate a single line of output in the statistics section of the
-# report.
-#
-proc statline {title value {extra {}}} {
-  set len [string length $title]
-  set dots [string repeat . [expr 50-$len]]
-  set len [string length $value]
-  set sp2 [string range {          } $len end]
-  if {$extra ne ""} {
-    set extra " $extra"
-  }
-  puts "$title$dots $value$sp2$extra"
-}
-
-# Generate a formatted percentage value for $num/$denom
-#
-proc percent {num denom {of {}}} {
-  if {$denom==0.0} {return ""}
-  set v [expr {$num*100.0/$denom}]
-  set of {}
-  if {$v==100.0 || $v<0.001 || ($v>1.0 && $v<99.0)} {
-    return [format {%5.1f%% %s} $v $of]
-  } elseif {$v<0.1 || $v>99.9} {
-    return [format {%7.3f%% %s} $v $of]
-  } else {
-    return [format {%6.2f%% %s} $v $of]
-  }
-}
-
-proc divide {num denom} {
-  if {$denom==0} {return 0.0}
-  return [format %.2f [expr double($num)/double($denom)]]
-}
-
-# Generate a subreport that covers some subset of the database.
-# the $where clause determines which subset to analyze.
-#
-proc subreport {title where showFrag} {
-  global pageSize file_pgcnt compressOverhead
-
-  # Query the in-memory database for the sum of various statistics 
-  # for the subset of tables/indices identified by the WHERE clause in
-  # $where. Note that even if the WHERE clause matches no rows, the
-  # following query returns exactly one row (because it is an aggregate).
-  #
-  # The results of the query are stored directly by SQLite into local 
-  # variables (i.e. $nentry, $payload etc.).
-  #
-  mem eval "
-    SELECT
-      int(sum(
-        CASE WHEN (is_without_rowid OR is_index) THEN nentry 
-             ELSE leaf_entries 
-        END
-      )) AS nentry,
-      int(sum(payload)) AS payload,
-      int(sum(ovfl_payload)) AS ovfl_payload,
-      max(mx_payload) AS mx_payload,
-      int(sum(ovfl_cnt)) as ovfl_cnt,
-      int(sum(leaf_pages)) AS leaf_pages,
-      int(sum(int_pages)) AS int_pages,
-      int(sum(ovfl_pages)) AS ovfl_pages,
-      int(sum(leaf_unused)) AS leaf_unused,
-      int(sum(int_unused)) AS int_unused,
-      int(sum(ovfl_unused)) AS ovfl_unused,
-      int(sum(gap_cnt)) AS gap_cnt,
-      int(sum(compressed_size)) AS compressed_size,
-      int(max(depth)) AS depth,
-      count(*) AS cnt
-    FROM space_used WHERE $where" {} {}
-
-  # Output the sub-report title, nicely decorated with * characters.
-  #
-  puts ""
-  titleline $title
-  puts ""
-
-  # Calculate statistics and store the results in TCL variables, as follows:
-  #
-  # total_pages: Database pages consumed.
-  # total_pages_percent: Pages consumed as a percentage of the file.
-  # storage: Bytes consumed.
-  # payload_percent: Payload bytes used as a percentage of $storage.
-  # total_unused: Unused bytes on pages.
-  # avg_payload: Average payload per btree entry.
-  # avg_fanout: Average fanout for internal pages.
-  # avg_unused: Average unused bytes per btree entry.
-  # avg_meta: Average metadata overhead per entry.
-  # ovfl_cnt_percent: Percentage of btree entries that use overflow pages.
-  #
-  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
-  set total_pages_percent [percent $total_pages $file_pgcnt]
-  set storage [expr {$total_pages*$pageSize}]
-  set payload_percent [percent $payload $storage {of storage consumed}]
-  set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
-  set avg_payload [divide $payload $nentry]
-  set avg_unused [divide $total_unused $nentry]
-  set total_meta [expr {$storage - $payload - $total_unused}]
-  set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}]
-  set meta_percent [percent $total_meta $storage {of metadata}]
-  set avg_meta [divide $total_meta $nentry]
-  if {$int_pages>0} {
-    # TODO: Is this formula correct?
-    set nTab [mem eval "
-      SELECT count(*) FROM (
-          SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0
-      )
-    "]
-    set avg_fanout [mem eval "
-      SELECT (sum(leaf_pages+int_pages)-$nTab)/sum(int_pages) FROM space_used
-          WHERE $where
-    "]
-    set avg_fanout [format %.2f $avg_fanout]
-  }
-  set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}]
-
-  # Print out the sub-report statistics.
-  #
-  statline {Percentage of total database} $total_pages_percent
-  statline {Number of entries} $nentry
-  statline {Bytes of storage consumed} $storage
-  if {$compressed_size!=$storage} {
-    set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
-    set pct [expr {$compressed_size*100.0/$storage}]
-    set pct [format {%5.1f%%} $pct]
-    statline {Bytes used after compression} $compressed_size $pct
-  }
-  statline {Bytes of payload} $payload $payload_percent
-  statline {Bytes of metadata} $total_meta $meta_percent
-  if {$cnt==1} {statline {B-tree depth} $depth}
-  statline {Average payload per entry} $avg_payload
-  statline {Average unused bytes per entry} $avg_unused
-  statline {Average metadata per entry} $avg_meta
-  if {[info exists avg_fanout]} {
-    statline {Average fanout} $avg_fanout
-  }
-  if {$showFrag && $total_pages>1} {
-    set fragmentation [percent $gap_cnt [expr {$total_pages-1}]]
-    statline {Non-sequential pages} $gap_cnt $fragmentation
-  }
-  statline {Maximum payload per entry} $mx_payload
-  statline {Entries that use overflow} $ovfl_cnt $ovfl_cnt_percent
-  if {$int_pages>0} {
-    statline {Index pages used} $int_pages
-  }
-  statline {Primary pages used} $leaf_pages
-  statline {Overflow pages used} $ovfl_pages
-  statline {Total pages used} $total_pages
-  if {$int_unused>0} {
-    set int_unused_percent [
-         percent $int_unused [expr {$int_pages*$pageSize}] {of index space}]
-    statline "Unused bytes on index pages" $int_unused $int_unused_percent
-  }
-  statline "Unused bytes on primary pages" $leaf_unused [
-     percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}]
-  statline "Unused bytes on overflow pages" $ovfl_unused [
-     percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}]
-  statline "Unused bytes on all pages" $total_unused [
-               percent $total_unused $storage {of all space}]
-  return 1
-}
-
-# Calculate the overhead in pages caused by auto-vacuum. 
-#
-# This procedure calculates and returns the number of pages used by the 
-# auto-vacuum 'pointer-map'. If the database does not support auto-vacuum,
-# then 0 is returned. The two arguments are the size of the database file in
-# pages and the page size used by the database (in bytes).
-proc autovacuum_overhead {filePages pageSize} {
-
-  # Set $autovacuum to non-zero for databases that support auto-vacuum.
-  set autovacuum [db one {PRAGMA auto_vacuum}]
-
-  # If the database is not an auto-vacuum database or the file consists
-  # of one page only then there is no overhead for auto-vacuum. Return zero.
-  if {0==$autovacuum || $filePages==1} {
-    return 0
-  }
-
-  # The number of entries on each pointer map page. The layout of the
-  # database file is one pointer-map page, followed by $ptrsPerPage other
-  # pages, followed by a pointer-map page etc. The first pointer-map page
-  # is the second page of the file overall.
-  set ptrsPerPage [expr double($pageSize/5)]
-
-  # Return the number of pointer map pages in the database.
-  return [expr wide(ceil( ($filePages-1.0)/($ptrsPerPage+1.0) ))]
-}
-
-
-# Calculate the summary statistics for the database and store the results
-# in TCL variables. They are output below. Variables are as follows:
-#
-# pageSize:      Size of each page in bytes.
-# file_bytes:    File size in bytes.
-# file_pgcnt:    Number of pages in the file.
-# file_pgcnt2:   Number of pages in the file (calculated).
-# av_pgcnt:      Pages consumed by the auto-vacuum pointer-map.
-# av_percent:    Percentage of the file consumed by auto-vacuum pointer-map.
-# inuse_pgcnt:   Data pages in the file.
-# inuse_percent: Percentage of pages used to store data.
-# free_pgcnt:    Free pages calculated as (<total pages> - <in-use pages>)
-# free_pgcnt2:   Free pages in the file according to the file header.
-# free_percent:  Percentage of file consumed by free pages (calculated).
-# free_percent2: Percentage of file consumed by free pages (header).
-# ntable:        Number of tables in the db.
-# nindex:        Number of indices in the db.
-# nautoindex:    Number of indices created automatically.
-# nmanindex:     Number of indices created manually.
-# user_payload:  Number of bytes of payload in table btrees 
-#                (not including sqlite_master)
-# user_percent:  $user_payload as a percentage of total file size.
-
-### The following, setting $file_bytes based on the actual size of the file
-### on disk, causes this tool to choke on zipvfs databases. So set it based
-### on the return of [PRAGMA page_count] instead.
-if 0 {
-  set file_bytes  [file size $file_to_analyze]
-  set file_pgcnt  [expr {$file_bytes/$pageSize}]
-}
-set file_pgcnt  [db one {PRAGMA page_count}]
-set file_bytes  [expr {$file_pgcnt * $pageSize}]
-
-set av_pgcnt    [autovacuum_overhead $file_pgcnt $pageSize]
-set av_percent  [percent $av_pgcnt $file_pgcnt]
-
-set sql {SELECT sum(leaf_pages+int_pages+ovfl_pages) FROM space_used}
-set inuse_pgcnt   [expr wide([mem eval $sql])]
-set inuse_percent [percent $inuse_pgcnt $file_pgcnt]
-
-set free_pgcnt    [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}]
-set free_percent  [percent $free_pgcnt $file_pgcnt]
-set free_pgcnt2   [db one {PRAGMA freelist_count}]
-set free_percent2 [percent $free_pgcnt2 $file_pgcnt]
-
-set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}]
-
-set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}]
-set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}]
-set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'}
-set nautoindex [db eval $sql]
-set nmanindex [expr {$nindex-$nautoindex}]
-
-# set total_payload [mem eval "SELECT sum(payload) FROM space_used"]
-set user_payload [mem one {SELECT int(sum(payload)) FROM space_used
-     WHERE NOT is_index AND name NOT LIKE 'sqlite_master'}]
-set user_percent [percent $user_payload $file_bytes]
-
-# Output the summary statistics calculated above.
-#
-puts "/** Disk-Space Utilization Report For $root_filename"
-puts ""
-statline {Page size in bytes} $pageSize
-statline {Pages in the whole file (measured)} $file_pgcnt
-statline {Pages in the whole file (calculated)} $file_pgcnt2
-statline {Pages that store data} $inuse_pgcnt $inuse_percent
-statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2
-statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent
-statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent
-statline {Number of tables in the database} $ntable
-statline {Number of indices} $nindex
-statline {Number of defined indices} $nmanindex
-statline {Number of implied indices} $nautoindex
-if {$isCompressed} {
-  statline {Size of uncompressed content in bytes} $file_bytes
-  set efficiency [percent $true_file_size $file_bytes]
-  statline {Size of compressed file on disk} $true_file_size $efficiency
-} else {
-  statline {Size of the file in bytes} $file_bytes
-}
-statline {Bytes of user payload stored} $user_payload $user_percent
-
-# Output table rankings
-#
-puts ""
-titleline "Page counts for all tables with their indices"
-puts ""
-mem eval {SELECT tblname, count(*) AS cnt, 
-              int(sum(int_pages+leaf_pages+ovfl_pages)) AS size
-          FROM space_used GROUP BY tblname ORDER BY size+0 DESC, tblname} {} {
-  statline [string toupper $tblname] $size [percent $size $file_pgcnt]
-}
-puts ""
-titleline "Page counts for all tables and indices separately"
-puts ""
-mem eval {
-  SELECT
-       upper(name) AS nm,
-       int(int_pages+leaf_pages+ovfl_pages) AS size
-    FROM space_used
-   ORDER BY size+0 DESC, name} {} {
-  statline $nm $size [percent $size $file_pgcnt]
-}
-if {$isCompressed} {
-  puts ""
-  titleline "Bytes of disk space used after compression"
-  puts ""
-  set csum 0
-  mem eval {SELECT tblname,
-                  int(sum(compressed_size)) +
-                         $compressOverhead*sum(int_pages+leaf_pages+ovfl_pages)
-                        AS csize
-          FROM space_used GROUP BY tblname ORDER BY csize+0 DESC, tblname} {} {
-    incr csum $csize
-    statline [string toupper $tblname] $csize [percent $csize $true_file_size]
-  }
-  set overhead [expr {$true_file_size - $csum}]
-  if {$overhead>0} {
-    statline {Header and free space} $overhead [percent $overhead $true_file_size]
-  }
-}
-
-# Output subreports
-#
-if {$nindex>0} {
-  subreport {All tables and indices} 1 0
-}
-subreport {All tables} {NOT is_index} 0
-if {$nindex>0} {
-  subreport {All indices} {is_index} 0
-}
-foreach tbl [mem eval {SELECT DISTINCT tblname name FROM space_used
-                       ORDER BY name}] {
-  set qn [quote $tbl]
-  set name [string toupper $tbl]
-  set n [mem eval {SELECT count(*) FROM space_used WHERE tblname=$tbl}]
-  if {$n>1} {
-    set idxlist [mem eval "SELECT name FROM space_used
-                            WHERE tblname='$qn' AND is_index
-                            ORDER BY 1"]
-    subreport "Table $name and all its indices" "tblname='$qn'" 0
-    subreport "Table $name w/o any indices" "name='$qn'" 1
-    if {[llength $idxlist]>1} {
-      subreport "Indices of table $name" "tblname='$qn' AND is_index" 0
-    }
-    foreach idx $idxlist {
-      set qidx [quote $idx]
-      subreport "Index [string toupper $idx] of table $name" "name='$qidx'" 1
-    }
-  } else {
-    subreport "Table $name" "name='$qn'" 1
-  }
-}
-
-# Output instructions on what the numbers above mean.
-#
-puts ""
-titleline Definitions
-puts {
-Page size in bytes
-
-    The number of bytes in a single page of the database file.  
-    Usually 1024.
-
-Number of pages in the whole file
-}
-puts "    The number of $pageSize-byte pages that go into forming the complete
-    database"
-puts {
-Pages that store data
-
-    The number of pages that store data, either as primary B*Tree pages or
-    as overflow pages.  The number at the right is the data pages divided by
-    the total number of pages in the file.
-
-Pages on the freelist
-
-    The number of pages that are not currently in use but are reserved for
-    future use.  The percentage at the right is the number of freelist pages
-    divided by the total number of pages in the file.
-
-Pages of auto-vacuum overhead
-
-    The number of pages that store data used by the database to facilitate
-    auto-vacuum. This is zero for databases that do not support auto-vacuum.
-
-Number of tables in the database
-
-    The number of tables in the database, including the SQLITE_MASTER table
-    used to store schema information.
-
-Number of indices
-
-    The total number of indices in the database.
-
-Number of defined indices
-
-    The number of indices created using an explicit CREATE INDEX statement.
-
-Number of implied indices
-
-    The number of indices used to implement PRIMARY KEY or UNIQUE constraints
-    on tables.
-
-Size of the file in bytes
-
-    The total amount of disk space used by the entire database files.
-
-Bytes of user payload stored
-
-    The total number of bytes of user payload stored in the database. The
-    schema information in the SQLITE_MASTER table is not counted when
-    computing this number.  The percentage at the right shows the payload
-    divided by the total file size.
-
-Percentage of total database
-
-    The amount of the complete database file that is devoted to storing
-    information described by this category.
-
-Number of entries
-
-    The total number of B-Tree key/value pairs stored under this category.
-
-Bytes of storage consumed
-
-    The total amount of disk space required to store all B-Tree entries
-    under this category.  The is the total number of pages used times
-    the pages size.
-
-Bytes of payload
-
-    The amount of payload stored under this category.  Payload is the data
-    part of table entries and the key part of index entries.  The percentage
-    at the right is the bytes of payload divided by the bytes of storage 
-    consumed.
-
-Bytes of metadata
-
-    The amount of formatting and structural information stored in the
-    table or index.  Metadata includes the btree page header, the cell pointer
-    array, the size field for each cell, the left child pointer or non-leaf
-    cells, the overflow pointers for overflow cells, and the rowid value for
-    rowid table cells.  In other words, metadata is everything that is neither
-    unused space nor content.  The record header in the payload is counted as
-    content, not metadata.
-
-Average payload per entry
-
-    The average amount of payload on each entry.  This is just the bytes of
-    payload divided by the number of entries.
-
-Average unused bytes per entry
-
-    The average amount of free space remaining on all pages under this
-    category on a per-entry basis.  This is the number of unused bytes on
-    all pages divided by the number of entries.
-
-Non-sequential pages
-
-    The number of pages in the table or index that are out of sequence.
-    Many filesystems are optimized for sequential file access so a small
-    number of non-sequential pages might result in faster queries,
-    especially for larger database files that do not fit in the disk cache.
-    Note that after running VACUUM, the root page of each table or index is
-    at the beginning of the database file and all other pages are in a
-    separate part of the database file, resulting in a single non-
-    sequential page.
-
-Maximum payload per entry
-
-    The largest payload size of any entry.
-
-Entries that use overflow
-
-    The number of entries that user one or more overflow pages.
-
-Total pages used
-
-    This is the number of pages used to hold all information in the current
-    category.  This is the sum of index, primary, and overflow pages.
-
-Index pages used
-
-    This is the number of pages in a table B-tree that hold only key (rowid)
-    information and no data.
-
-Primary pages used
-
-    This is the number of B-tree pages that hold both key and data.
-
-Overflow pages used
-
-    The total number of overflow pages used for this category.
-
-Unused bytes on index pages
-
-    The total number of bytes of unused space on all index pages.  The
-    percentage at the right is the number of unused bytes divided by the
-    total number of bytes on index pages.
-
-Unused bytes on primary pages
-
-    The total number of bytes of unused space on all primary pages.  The
-    percentage at the right is the number of unused bytes divided by the
-    total number of bytes on primary pages.
-
-Unused bytes on overflow pages
-
-    The total number of bytes of unused space on all overflow pages.  The
-    percentage at the right is the number of unused bytes divided by the
-    total number of bytes on overflow pages.
-
-Unused bytes on all pages
-
-    The total number of bytes of unused space on all primary and overflow 
-    pages.  The percentage at the right is the number of unused bytes 
-    divided by the total number of bytes.
-}
-
-# Output a dump of the in-memory database. This can be used for more
-# complex offline analysis.
-#
-titleline {}
-puts "The entire text of this report can be sourced into any SQL database"
-puts "engine for further analysis.  All of the text above is an SQL comment."
-puts "The data used to generate this report follows:"
-puts "*/"
-puts "BEGIN;"
-puts $tabledef
-unset -nocomplain x
-mem eval {SELECT * FROM space_used} x {
-  puts -nonewline "INSERT INTO space_used VALUES"
-  set sep (
-  foreach col $x(*) {
-    set v $x($col)
-    if {$v=="" || ![string is double $v]} {set v '[quote $v]'}
-    puts -nonewline $sep$v
-    set sep ,
-  }
-  puts ");"
-}
-puts "COMMIT;"
-
-} err]} {
-  puts "ERROR: $err"
-  puts $errorInfo
-  exit 1
-}
+INCLUDE $ROOT/tool/spaceanal.tcl
 END_STRING
 ;
 }

From 835cd436e1e0a964d84034a88303a1b4c0e8083c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 14 Oct 2017 19:54:06 +0000
Subject: [PATCH 190/270] Updates to the Makefiles for MSVC.

FossilOrigin-Name: ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
---
 Makefile.msc          |  4 ++--
 autoconf/Makefile.msc |  5 +++--
 manifest              | 16 ++++++++--------
 manifest.uuid         |  2 +-
 4 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index ce867ba3f3..2804c25ab4 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1984,8 +1984,8 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c
 
-shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
-	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
 
 
 # Rules to build the extension objects.
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index f535d1450a..e0f7ad8615 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -928,6 +928,7 @@ LIBRESOBJS =
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 !ENDIF
 
 
@@ -957,8 +958,8 @@ sqlite3.def:	Replace.exe $(LIBOBJ)
 		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def
 
-$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 
diff --git a/manifest b/manifest
index 2599e89765..1ce91e738e 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Improved\sability\sto\sgenerate\sstand-alone\sprogram\susing\sTCL\sand\sSQLite\sby\ncompiling\swith\s-DTCLSH_INIT_PROC=name\sto\scause\sthe\sTCL\sinterpreter\sto\sbe\ninitialized\susing\sprocedure\sname().\s\sBoth\ssqlite3_analyzer\sand\stestfixture\nare\snow\sbuilt\sthis\sway.
-D 2017-10-13T20:14:06.865
+C Updates\sto\sthe\sMakefiles\sfor\sMSVC.
+D 2017-10-14T19:54:06.773
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc a341cc7d737f596a1074e47fc9eb17bde689413d891e3ef2f26e98126da950a4
+F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
-F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd
+F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 298a3fddec459c4fd2b840bd363239dc627f1dda90e2d5e478846cb895a8ad82
-R df005e0c0c347d89af378f70d82f057b
-U drh
-Z 8b50e8b5594af446543ab2fc55fee9a2
+P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4
+R 49fa7e54acb9ad8a392e6c25fe8116d5
+U mistachkin
+Z 3270bf7f35234fd560a264ffe40cbf35
diff --git a/manifest.uuid b/manifest.uuid
index d905d370b6..63b3b0cf2b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4
\ No newline at end of file
+ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
\ No newline at end of file

From 82cae9ffd62c1375adef56d9e0d43c099691c797 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 14 Oct 2017 19:58:37 +0000
Subject: [PATCH 191/270] Updates to the Makefiles for MSVC.  Cherrypick of
 [ac8786f3f9f35cb6].

FossilOrigin-Name: 92eb721faefcdd8396072722d3e4d7ca41b860b306e4bb0f0191dde8f30d0add
---
 Makefile.msc          |  4 ++--
 autoconf/Makefile.msc |  4 ++--
 manifest              | 17 +++++++++--------
 manifest.uuid         |  2 +-
 4 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 4c3c67f94d..2e18c50d47 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1976,8 +1976,8 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c
 
-shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
-	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl > shell.c
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
 
 
 # Rules to build the extension objects.
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index f535d1450a..46f7c7a25b 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -957,8 +957,8 @@ sqlite3.def:	Replace.exe $(LIBOBJ)
 		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
 		| sort >> sqlite3.def
 
-$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
-	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
+$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 
diff --git a/manifest b/manifest
index b720d2abb5..9e350ba263 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sa\sharmless\scompiler\swarning\sabout\san\sunused\sparameter.
-D 2017-10-12T14:18:38.182
+C Updates\sto\sthe\sMakefiles\sfor\sMSVC.\s\sCherrypick\sof\s[ac8786f3f9f35cb6].
+D 2017-10-14T19:58:37.221
 F Makefile.in 9c9f4dea3f622464cba9768501aceca187d2bbae10b60bf420b531cd776fe5c0
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 3f96a87fb271b06aede7e304234cce096edd3d5ad76507ccc4716b20511a3b20
+F Makefile.msc 4a7ebdaacec1abd415d2847c8f5b42053c5963294e925672d43d2972a5bb2cb6
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
-F autoconf/Makefile.msc 55328895d224f1b81298224929a16b8601baa75f9d7badd493224f68a22700dd
+F autoconf/Makefile.msc b045158822d2320d5551cb7291ecc685dd6123324b928490691deed71274c305
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578
@@ -1656,7 +1656,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c643ace24c1936f9d2b16bd6d1d13cf08070dfe5b094208b638924e904915339
-R 6b8c8a980bca85216c2bbb28216f8b36
-U drh
-Z ee2d81a451eaa5a0334a9c3d00838821
+P de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0
+Q +ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
+R 8e422c1ee06cd57625ef547ccd862b5a
+U mistachkin
+Z 0ed36b5b53069fc1c0ccb6054caf79bd
diff --git a/manifest.uuid b/manifest.uuid
index 0b314a09be..524d43870c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-de20133d44773f0b3b8869db5c1cb2a90f0426a54c7f40d12a930003343ad8e0
\ No newline at end of file
+92eb721faefcdd8396072722d3e4d7ca41b860b306e4bb0f0191dde8f30d0add
\ No newline at end of file

From 47b1d68f25f2c921497c6a642a111d2d8e7d00cc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 15 Oct 2017 22:16:25 +0000
Subject: [PATCH 192/270] In the query planner, do not consider index X to be a
 proper subset of index Y if X is a covering index but Y is not.

FossilOrigin-Name: ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d
---
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 src/where.c        | 17 ++++++++++-------
 test/analyze9.test |  5 ++++-
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 1ce91e738e..b77b1484ce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\sMakefiles\sfor\sMSVC.
-D 2017-10-14T19:54:06.773
+C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot.
+D 2017-10-15T22:16:25.229
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5
+F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4
@@ -568,7 +568,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4
 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f
 F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f
 F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82
-F test/analyze9.test b817b8e798315fc65b820a5463f73ad5f48ed8dd
+F test/analyze9.test dac0bdc7eab965b9ad639ca83564d98717aaf13ce5a776f23d9a3680238cecd8
 F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a
 F test/analyzeB.test a4c1c3048f6d9e090eb76e83eecb18bcf6d31a70
 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d65d1f297ddb07b799ff5b2e560575fc59a6fa74c752269cc85ab84348fb7da4
-R 49fa7e54acb9ad8a392e6c25fe8116d5
-U mistachkin
-Z 3270bf7f35234fd560a264ffe40cbf35
+P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
+R 88b26813bcaf999746e4cf55503b4797
+U drh
+Z fe6d8478e3f31cb2dfbec6ce858d4ad8
diff --git a/manifest.uuid b/manifest.uuid
index 63b3b0cf2b..12179a84a0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
\ No newline at end of file
+ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 5a3d44f9ee..95bbdb157e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1885,16 +1885,15 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
 ** Return TRUE if all of the following are true:
 **
 **   (1)  X has the same or lower cost that Y
-**   (2)  X is a proper subset of Y
-**   (3)  X skips at least as many columns as Y
-**
-** By "proper subset" we mean that X uses fewer WHERE clause terms
-** than Y and that every WHERE clause term used by X is also used
-** by Y.
+**   (2)  X users fewer WHERE clause terms than Y
+**   (3)  Every WHERE clause term used by X is also used by Y
+**   (4)  X skips at least as many columns as Y
+**   (5)  If X is a covering index, than Y is too
 **
+** Conditions (2) and (3) mean that X is a "proper subset" of Y.
 ** If X is a proper subset of Y then Y is a better choice and ought
 ** to have a lower cost.  This routine returns TRUE when that cost 
-** relationship is inverted and needs to be adjusted.  The third rule
+** relationship is inverted and needs to be adjusted.  Constraint (4)
 ** was added because if X uses skip-scan less than Y it still might
 ** deserve a lower cost even if it is a proper subset of Y.
 */
@@ -1918,6 +1917,10 @@ static int whereLoopCheaperProperSubset(
     }
     if( j<0 ) return 0;  /* X not a subset of Y since term X[i] not used by Y */
   }
+  if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 
+   && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
+    return 0;  /* Constraint (5) */
+  }
   return 1;  /* All conditions meet */
 }
 
diff --git a/test/analyze9.test b/test/analyze9.test
index 918577bd19..7c69806f89 100644
--- a/test/analyze9.test
+++ b/test/analyze9.test
@@ -1052,8 +1052,11 @@ do_execsql_test 23.0 {
 do_eqp_test 23.1 {
   SELECT * FROM t4 WHERE 
     (e=1 AND b='xyz' AND c='zyx' AND a<'AEA') AND f<300
+  -- Formerly used index i41.  But i41 is not a covering index whereas
+  -- the PRIMARY KEY is a covering index, and so as of 2017-10-15, the
+  -- PRIMARY KEY is preferred.
 } {
-  0 0 0 {SEARCH TABLE t4 USING INDEX i41 (e=? AND c=? AND b=? AND a<?)}
+  0 0 0 {SEARCH TABLE t4 USING PRIMARY KEY (c=? AND b=? AND a<?)}
 }
 do_eqp_test 23.2 {
   SELECT * FROM t4 WHERE 

From 989d7278b0821e6e8550275ad741c08998385a61 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 16 Oct 2017 11:50:12 +0000
Subject: [PATCH 193/270] Improved comments, including a typo fix, on the
 whereLoopCheaperProperSubset() function.

FossilOrigin-Name: 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/where.c   |  6 ++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index b77b1484ce..ab44e57d39 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\squery\splanner,\sdo\snot\sconsider\sindex\sX\sto\sbe\sa\sproper\ssubset\sof\s\nindex\sY\sif\sX\sis\sa\scovering\sindex\sbut\sY\sis\snot.
-D 2017-10-15T22:16:25.229
+C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction.
+D 2017-10-16T11:50:12.951
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -544,7 +544,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c 248f0dc6085b37b4b2ccad2de06b90f1a48b1932403ab0ad9100a1f2a6990763
+F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ac8786f3f9f35cb6fa72c65f5ca41cc6659f4702cd29ab0c9dcf04d3d1189090
-R 88b26813bcaf999746e4cf55503b4797
+P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d
+R 593b5394fda7f5c3a7e7e5312c0a0c37
 U drh
-Z fe6d8478e3f31cb2dfbec6ce858d4ad8
+Z d6213a58e95c0fa7e25d95c2efa3d8ac
diff --git a/manifest.uuid b/manifest.uuid
index 12179a84a0..3281f881f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d
\ No newline at end of file
+700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 95bbdb157e..6021edf47f 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1885,7 +1885,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
 ** Return TRUE if all of the following are true:
 **
 **   (1)  X has the same or lower cost that Y
-**   (2)  X users fewer WHERE clause terms than Y
+**   (2)  X uses fewer WHERE clause terms than Y
 **   (3)  Every WHERE clause term used by X is also used by Y
 **   (4)  X skips at least as many columns as Y
 **   (5)  If X is a covering index, than Y is too
@@ -1895,7 +1895,9 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
 ** to have a lower cost.  This routine returns TRUE when that cost 
 ** relationship is inverted and needs to be adjusted.  Constraint (4)
 ** was added because if X uses skip-scan less than Y it still might
-** deserve a lower cost even if it is a proper subset of Y.
+** deserve a lower cost even if it is a proper subset of Y.  Constraint (5)
+** was added because a covering index probably deserves to have a lower cost
+** than a non-covering index even if it is a proper subset.
 */
 static int whereLoopCheaperProperSubset(
   const WhereLoop *pX,       /* First WhereLoop to compare */

From f30d345241ac0d260c5a8034b1366c5bee596ee3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 17 Oct 2017 13:44:46 +0000
Subject: [PATCH 194/270] Fix the ".check" command of the command-line shell to
 return 1 (error) rather than 2 (exit) on a test failure.

FossilOrigin-Name: e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index ab44e57d39..de1b01a07e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\scomments,\sincluding\sa\stypo\sfix,\son\sthe\swhereLoopCheaperProperSubset()\nfunction.
-D 2017-10-16T11:50:12.951
+C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure.
+D 2017-10-17T13:44:46.075
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -462,7 +462,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
-F src/shell.c.in 5446de0a90c15d713bbdb5827cf57ec30d1c3497097f39ec2c2e874dcca34ca3
+F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ee31c04353cd75ea4bbadee2994c30d3808b696a4f680187502d104902988a5d
-R 593b5394fda7f5c3a7e7e5312c0a0c37
+P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
+R 7b11fa4b75f3d72b968740b4ad577de5
 U drh
-Z d6213a58e95c0fa7e25d95c2efa3d8ac
+Z eb279a84da6335af377cd536eb14b148
diff --git a/manifest.uuid b/manifest.uuid
index 3281f881f9..cd80d78996 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
\ No newline at end of file
+e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index b62c055076..8f5ed59e93 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4248,7 +4248,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       utf8_printf(stderr,
                  "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
                  p->zTestcase, azArg[1], zRes);
-      rc = 2;
+      rc = 1;
     }else{
       utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase);
       p->nCheck++;

From f7fc4c22f45adcf20fa88febad0a0e046a04f9ae Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 17 Oct 2017 18:33:22 +0000
Subject: [PATCH 195/270] The source code files generated for the
 'sqlite3-all.c' target should not have Windows line-endings.

FossilOrigin-Name: 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668
---
 manifest                | 14 +++++++-------
 manifest.uuid           |  2 +-
 tool/split-sqlite3c.tcl |  2 ++
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index de1b01a07e..d6e49a102d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure.
-D 2017-10-17T13:44:46.075
+C The\ssource\scode\sfiles\sgenerated\sfor\sthe\s'sqlite3-all.c'\starget\sshould\snot\shave\sWindows\sline-endings.
+D 2017-10-17T18:33:22.646
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -1630,7 +1630,7 @@ F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
-F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
+F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
-R 7b11fa4b75f3d72b968740b4ad577de5
-U drh
-Z eb279a84da6335af377cd536eb14b148
+P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
+R d1f05de03c82774074759a8848af421f
+U mistachkin
+Z cf3e5ebd3225c433ae20a29b56e94ef7
diff --git a/manifest.uuid b/manifest.uuid
index cd80d78996..4c5b7a09c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
\ No newline at end of file
+6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668
\ No newline at end of file
diff --git a/tool/split-sqlite3c.tcl b/tool/split-sqlite3c.tcl
index 287b752828..230e3f2549 100644
--- a/tool/split-sqlite3c.tcl
+++ b/tool/split-sqlite3c.tcl
@@ -15,6 +15,7 @@ set END   {^/\*+ End of %s \*+/}
 
 set in [open sqlite3.c]
 set out1 [open sqlite3-all.c w]
+fconfigure $out1 -translation lf
 
 # Copy the header from sqlite3.c into sqlite3-all.c
 #
@@ -48,6 +49,7 @@ proc write_one_file {content} {
   global filecnt
   incr filecnt
   set out [open sqlite3-$filecnt.c w]
+  fconfigure $out -translation lf
   puts -nonewline $out $content
   close $out
   puts $::out1 "#include \"sqlite3-$filecnt.c\""

From af124faf3e6e6284161b84e68f7e13235bd6afb4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 18 Oct 2017 14:29:36 +0000
Subject: [PATCH 196/270] Avoid using snprintf as a member variable of a struct
 in sqlite3ext.h. This causes problems on OSX.

FossilOrigin-Name: b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c
---
 manifest         | 17 ++++++++++-------
 manifest.uuid    |  2 +-
 src/sqlite3ext.h |  4 ++--
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index de1b01a07e..365c4e7bb8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s".check"\scommand\sof\sthe\scommand-line\sshell\sto\sreturn\s1\s(error)\nrather\sthan\s2\s(exit)\son\sa\stest\sfailure.
-D 2017-10-17T13:44:46.075
+C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX.
+D 2017-10-18T14:29:36.881
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -465,7 +465,7 @@ F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
-F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47
+F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
@@ -1664,7 +1664,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 700a3c694438ca6cca185d0097f24799e82717ef38cb47bd83666c80f0e3cb2f
-R 7b11fa4b75f3d72b968740b4ad577de5
-U drh
-Z eb279a84da6335af377cd536eb14b148
+P e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
+R af758af6d9338e0da5d96870c08db747
+T *branch * sqlite3ext-fix
+T *sym-sqlite3ext-fix *
+T -sym-trunk *
+U dan
+Z d17bbc38b957a75c44d151990964eb3a
diff --git a/manifest.uuid b/manifest.uuid
index cd80d78996..73db2c4c9b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e2af0cc6ef5fafc785b8b73f51995901f50822972dcfb632494e22402596186d
\ No newline at end of file
+b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c
\ No newline at end of file
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 4aade278bd..d1d2c574ae 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -134,7 +134,7 @@ struct sqlite3_api_routines {
   int  (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
                          const char*,const char*),void*);
   void  (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
-  char * (*snprintf)(int,char*,const char*,...);
+  char * (*xsnprintf)(int,char*,const char*,...);
   int  (*step)(sqlite3_stmt*);
   int  (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
                                 char const**,char const**,int*,int*,int*);
@@ -418,7 +418,7 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_rollback_hook          sqlite3_api->rollback_hook
 #define sqlite3_set_authorizer         sqlite3_api->set_authorizer
 #define sqlite3_set_auxdata            sqlite3_api->set_auxdata
-#define sqlite3_snprintf               sqlite3_api->snprintf
+#define sqlite3_snprintf               sqlite3_api->xsnprintf
 #define sqlite3_step                   sqlite3_api->step
 #define sqlite3_table_column_metadata  sqlite3_api->table_column_metadata
 #define sqlite3_thread_cleanup         sqlite3_api->thread_cleanup

From f2f72a0f9fe7125285f233011bb10257fa5a6a88 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 19 Oct 2017 15:17:38 +0000
Subject: [PATCH 197/270] Take extra care to avoid an OOB read caused by a
 corrupt b-tree page.

FossilOrigin-Name: 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
---
 manifest               |  18 +++----
 manifest.uuid          |   2 +-
 src/btree.c            |  13 +++--
 test/corruptK.test     | 109 +++++++++++++++++++++++++++++++++++++++++
 test/permutations.test |   2 +-
 5 files changed, 129 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 75ecef5321..5c8ae4d13c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\susing\ssnprintf\sas\sa\smember\svariable\sof\sa\sstruct\sin\ssqlite3ext.h.\sThis\ncauses\sproblems\son\sOSX.\sSimilar\sto\sfix\s[a1fd3aa8].
-D 2017-10-18T15:02:01.361
+C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage.
+D 2017-10-19T15:17:38.752
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -403,7 +403,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 8565b061a6a6fad850230c73d6a7a8ffb88f3370e3352a8689a9a672160c5cc5
+F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673
@@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51
 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454
 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4
 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
-F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe
+F test/corruptK.test 251ef631d095e882d455d2183961fa9ba879b4156e18e96c5d2b84aa7ef5f7a9
 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8
 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c
 F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c
@@ -1105,7 +1105,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 53fb87f0a345259f884fa8b5fc62ee327b20be378ebe5149833b1b28c58a42f6
+F test/permutations.test 490e3333b9b1aefb7ebc6e9ab2ae0e382b7dd8713ccc4a2786b0f75467c2ab6b
 F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1664,8 +1664,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6a08c43431be18a08bdcbf33d327513f72fff72dac5d02103dab8399d8c3d668 b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c
-R bae812ccbd328c244f2fa888e3ba625e
-T +closed b066d5a69f5391b2cef04b4e288f7d33e63a4ff54325b7e9fbd6aa42529c2f3c
+P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46
+R 14641e8822b464d39bf472f899e19240
+T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a
 U dan
-Z 4879d15d8231001302e10b2e0a3c2891
+Z 576e55698c218b4128bbad5de39dc1b0
diff --git a/manifest.uuid b/manifest.uuid
index 2f3fcd4b40..26a2375f60 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46
\ No newline at end of file
+04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 89222197bc..ddcb6cfd35 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4824,7 +4824,7 @@ static const void *fetchPayload(
   BtCursor *pCur,      /* Cursor pointing to entry to read from */
   u32 *pAmt            /* Write the number of available bytes here */
 ){
-  u32 amt;
+  int amt;
   assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage);
   assert( pCur->eState==CURSOR_VALID );
   assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
@@ -4833,9 +4833,14 @@ static const void *fetchPayload(
   assert( pCur->info.nSize>0 );
   assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB );
   assert( pCur->info.pPayload<pCur->pPage->aDataEnd ||CORRUPT_DB);
-  amt = (int)(pCur->pPage->aDataEnd - pCur->info.pPayload);
-  if( pCur->info.nLocal<amt ) amt = pCur->info.nLocal;
-  *pAmt = amt;
+  amt = pCur->info.nLocal;
+  if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){
+    /* There is too little space on the page for the expected amount
+    ** of local content. Database must be corrupt. */
+    assert( CORRUPT_DB );
+    amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload));
+  }
+  *pAmt = (u32)amt;
   return (void*)pCur->info.pPayload;
 }
 
diff --git a/test/corruptK.test b/test/corruptK.test
index b20c2d8bf0..b921176974 100644
--- a/test/corruptK.test
+++ b/test/corruptK.test
@@ -107,6 +107,115 @@ do_catchsql_test 2.3 {
   INSERT INTO t1 VALUES(randomblob(900));
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+
+proc hex2blob {hex} {
+  # Split on newlines:
+  set bytes [list]
+  foreach l [split $hex "\n"] {
+    if {[string is space $l]} continue
+    set L [list]
+    foreach b [split $l] {
+      if {[string is xdigit $b] && [string length $b]==2} { 
+        lappend L [expr "0x$b"]
+      }
+    }
+    if {[llength $L]!=16} {
+      error "Badly formed hex (1)"
+    }
+    set bytes [concat $bytes $L]
+  }
+
+  binary format c* $bytes
+}
+
+reset_db
+db func hex2blob hex2blob
+
+do_execsql_test 3.1 {
+  PRAGMA page_size=1024;
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE t2(a, b, c);
+  CREATE TABLE t3(a, b, c);
+  CREATE TABLE t4(a, b, c);
+  CREATE TABLE t5(a, b, c);
+}
+
+do_execsql_test 3.2 {
+  UPDATE sqlite_dbpage SET data = hex2blob('
+ 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
+ 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @  ..>.....
+ 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................
+ 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................
+ 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8..
+ 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>.
+ 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-..............
+ 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................
+ 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0...
+ 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................
+ 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........
+ 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................
+ 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p.
+ 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h..........
+ 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................
+ 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................
+ 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p...........
+ 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+ 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese
+ 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE
+ 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor
+ 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s .  $          
+ 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal 
+ 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text)
+  ') WHERE pgno=1
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 3.3 {
+  PRAGMA integrity_check;
+} {1 {database disk image is malformed}}
 
 
 
diff --git a/test/permutations.test b/test/permutations.test
index 1e04265668..76fb72ace5 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -194,7 +194,7 @@ test_suite "valgrind" -prefix "" -description {
 } -files [
   test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* wal.test \
               shell*.test crash8.test atof1.test selectG.test \
-              tkt-fc62af4523.test numindex1.test
+              tkt-fc62af4523.test numindex1.test corruptK.test
 ] -initialize {
   set ::G(valgrind) 1
 } -shutdown {

From 316ab93e2c92526d0fd2b42f53b75be32e6a0428 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 21 Oct 2017 13:29:26 +0000
Subject: [PATCH 198/270] Remove unnecessary "#if SQLITE_MAX_COLUMN". 
 SQLITE_MAX_COLUMN is always defined.

FossilOrigin-Name: 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a
---
 manifest      | 19 +++++++++----------
 manifest.uuid |  2 +-
 src/build.c   |  2 --
 src/resolve.c |  4 ----
 src/select.c  |  2 --
 5 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index 5c8ae4d13c..cd878b65fe 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage.
-D 2017-10-19T15:17:38.752
+C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined.
+D 2017-10-21T13:29:26.479
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -406,7 +406,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 6ffe76970aeee4bc94e60cf0138269e67109061a853e13098c38a904dd66e673
+F src/build.c e24b61144f9c9b15c4aa05954514190361061da721e56dcd1af6e0e945ee5909
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -459,9 +459,9 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
+F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 42aca61e739c405ddd8a1b702977a7743c7d52a94885f7c5596bd7e73e6bff18
+F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1664,8 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46
-R 14641e8822b464d39bf472f899e19240
-T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a
-U dan
-Z 576e55698c218b4128bbad5de39dc1b0
+P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
+R 3fe93e66991cd1a89b7cde18365d4335
+U drh
+Z 82fbf688aff072892ed2c0d525c25911
diff --git a/manifest.uuid b/manifest.uuid
index 26a2375f60..14db0bd9b2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
\ No newline at end of file
+6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index cb8c374d1a..a9428104a3 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1063,12 +1063,10 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){
   Column *pCol;
   sqlite3 *db = pParse->db;
   if( (p = pParse->pNewTable)==0 ) return;
-#if SQLITE_MAX_COLUMN
   if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){
     sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName);
     return;
   }
-#endif
   z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2);
   if( z==0 ) return;
   memcpy(z, pName->z, pName->n);
diff --git a/src/resolve.c b/src/resolve.c
index 78f37512a2..4a1e8284c8 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -959,12 +959,10 @@ static int resolveCompoundOrderBy(
   pOrderBy = pSelect->pOrderBy;
   if( pOrderBy==0 ) return 0;
   db = pParse->db;
-#if SQLITE_MAX_COLUMN
   if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
     sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause");
     return 1;
   }
-#endif
   for(i=0; i<pOrderBy->nExpr; i++){
     pOrderBy->a[i].done = 0;
   }
@@ -1056,12 +1054,10 @@ int sqlite3ResolveOrderGroupBy(
   struct ExprList_item *pItem;
 
   if( pOrderBy==0 || pParse->db->mallocFailed ) return 0;
-#if SQLITE_MAX_COLUMN
   if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
     sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType);
     return 1;
   }
-#endif
   pEList = pSelect->pEList;
   assert( pEList!=0 );  /* sqlite3SelectNew() guarantees this */
   for(i=0, pItem=pOrderBy->a; i<pOrderBy->nExpr; i++, pItem++){
diff --git a/src/select.c b/src/select.c
index 6f524fa2e1..9639cfde1a 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4596,12 +4596,10 @@ static int selectExpander(Walker *pWalker, Select *p){
     sqlite3ExprListDelete(db, pEList);
     p->pEList = pNew;
   }
-#if SQLITE_MAX_COLUMN
   if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
     sqlite3ErrorMsg(pParse, "too many columns in result set");
     return WRC_Abort;
   }
-#endif
   return WRC_Continue;
 }
 

From 6fe3733ba97193227a395c7796bb9ca0d6f82134 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 21 Oct 2017 14:17:31 +0000
Subject: [PATCH 199/270] Catch and avoid a 16-bit integer overflow on the
 number of columns in a common table expression.  This fixes a problem found
 by OSS-Fuzz.  The test case is in TH3.

FossilOrigin-Name: 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index cd878b65fe..da985ae44f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sunnecessary\s"#if\sSQLITE_MAX_COLUMN".\s\sSQLITE_MAX_COLUMN\sis\salways\ndefined.
-D 2017-10-21T13:29:26.479
+C Catch\sand\savoid\sa\s16-bit\sinteger\soverflow\son\sthe\snumber\sof\scolumns\sin\sa\ncommon\stable\sexpression.\s\sThis\sfixes\sa\sproblem\sfound\sby\sOSS-Fuzz.\s\sThe\ntest\scase\sis\sin\sTH3.
+D 2017-10-21T14:17:31.555
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -461,7 +461,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 78b81b0d0f8ba2445e4de5ca3c97a9fd317240a9c5e4994887d4ae8a1d5a3367
+F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
-R 3fe93e66991cd1a89b7cde18365d4335
+P 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a
+R 201775ba1e14aa2d5f8d0e57defde230
 U drh
-Z 82fbf688aff072892ed2c0d525c25911
+Z bad3386d6737a5472f5b033811b6e996
diff --git a/manifest.uuid b/manifest.uuid
index 14db0bd9b2..60279d9206 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a
\ No newline at end of file
+6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 9639cfde1a..0e2328120b 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1689,6 +1689,7 @@ int sqlite3ColumnsFromExprList(
     nCol = pEList->nExpr;
     aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol);
     testcase( aCol==0 );
+    if( nCol>32767 ) nCol = 32767;
   }else{
     nCol = 0;
     aCol = 0;

From 9499b4a6143b9464fa27e25dddd6a5301ab352de Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sun, 22 Oct 2017 07:57:29 +0000
Subject: [PATCH 200/270] Avoid running tests that use sqlite_dbpage with
 SQLITE_OMIT_VIRTUAL_TABLE builds.

FossilOrigin-Name: 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528
---
 manifest           |  14 +--
 manifest.uuid      |   2 +-
 test/corruptK.test | 210 +++++++++++++++++++++++----------------------
 3 files changed, 115 insertions(+), 111 deletions(-)

diff --git a/manifest b/manifest
index da985ae44f..07fa1c88b3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Catch\sand\savoid\sa\s16-bit\sinteger\soverflow\son\sthe\snumber\sof\scolumns\sin\sa\ncommon\stable\sexpression.\s\sThis\sfixes\sa\sproblem\sfound\sby\sOSS-Fuzz.\s\sThe\ntest\scase\sis\sin\sTH3.
-D 2017-10-21T14:17:31.555
+C Avoid\srunning\stests\sthat\suse\ssqlite_dbpage\swith\sSQLITE_OMIT_VIRTUAL_TABLE\nbuilds.
+D 2017-10-22T07:57:29.038
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51
 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454
 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4
 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
-F test/corruptK.test 251ef631d095e882d455d2183961fa9ba879b4156e18e96c5d2b84aa7ef5f7a9
+F test/corruptK.test d93f5797bf0555a47dcc81bc9727cb72fa7c8ac348c25f42937573cd29bb3850
 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8
 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c
 F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ec82acde81a46a75ed5931fc7dd813f2523753106ad7b8f0b544b9da9824d5a
-R 201775ba1e14aa2d5f8d0e57defde230
-U drh
-Z bad3386d6737a5472f5b033811b6e996
+P 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c
+R 2ad0f857ce1c547a47a99955708ad02e
+U dan
+Z 31291a1d9ccb37db331d41df009b953d
diff --git a/manifest.uuid b/manifest.uuid
index 60279d9206..321fef9696 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c
\ No newline at end of file
+7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528
\ No newline at end of file
diff --git a/test/corruptK.test b/test/corruptK.test
index b921176974..05465a9b5f 100644
--- a/test/corruptK.test
+++ b/test/corruptK.test
@@ -109,113 +109,117 @@ do_catchsql_test 2.3 {
 
 #-------------------------------------------------------------------------
 
-proc hex2blob {hex} {
-  # Split on newlines:
-  set bytes [list]
-  foreach l [split $hex "\n"] {
-    if {[string is space $l]} continue
-    set L [list]
-    foreach b [split $l] {
-      if {[string is xdigit $b] && [string length $b]==2} { 
-        lappend L [expr "0x$b"]
+ifcapable vtab {
+
+  proc hex2blob {hex} {
+    # Split on newlines:
+    set bytes [list]
+    foreach l [split $hex "\n"] {
+      if {[string is space $l]} continue
+      set L [list]
+      foreach b [split $l] {
+        if {[string is xdigit $b] && [string length $b]==2} { 
+          lappend L [expr "0x$b"]
+        }
       }
+      if {[llength $L]!=16} {
+        error "Badly formed hex (1)"
+      }
+      set bytes [concat $bytes $L]
     }
-    if {[llength $L]!=16} {
-      error "Badly formed hex (1)"
-    }
-    set bytes [concat $bytes $L]
+  
+    binary format c* $bytes
   }
+  
+  reset_db
+  db func hex2blob hex2blob
+  
+  do_execsql_test 3.1 {
+    PRAGMA page_size=1024;
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(a, b, c);
+    CREATE TABLE t3(a, b, c);
+    CREATE TABLE t4(a, b, c);
+    CREATE TABLE t5(a, b, c);
+  }
+  
+  do_execsql_test 3.2 {
+    UPDATE sqlite_dbpage SET data = hex2blob('
+   000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
+   010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @  ..>.....
+   020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................
+   030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................
+   040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8..
+   050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>.
+   060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-..............
+   070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................
+   0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0...
+   190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................
+   1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........
+   240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................
+   2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p.
+   320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h..........
+   360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................
+   370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................
+   380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p...........
+   390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
+   3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese
+   3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE
+   3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor
+   3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s .  $          
+   3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal 
+   3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text)
+    ') WHERE pgno=1
+  }
+  
+  db close
+  sqlite3 db test.db
+  
+  do_catchsql_test 3.3 {
+    PRAGMA integrity_check;
+  } {1 {database disk image is malformed}}
 
-  binary format c* $bytes
-}
-
-reset_db
-db func hex2blob hex2blob
-
-do_execsql_test 3.1 {
-  PRAGMA page_size=1024;
-  CREATE TABLE t1(a, b, c);
-  CREATE TABLE t2(a, b, c);
-  CREATE TABLE t3(a, b, c);
-  CREATE TABLE t4(a, b, c);
-  CREATE TABLE t5(a, b, c);
-}
-
-do_execsql_test 3.2 {
-  UPDATE sqlite_dbpage SET data = hex2blob('
- 000: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
- 010: 04 00 01 01 20 40 20 20 00 00 3e d9 00 00 00 06 .... @  ..>.....
- 020: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................
- 030: 0f 00 00 00 00 00 00 00 00 00 00 01 00 00 83 00 ................
- 040: 00 00 00 00 00 00 00 00 00 00 00 00 00 38 00 00 .............8..
- 050: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3e d9 ..............>.
- 060: 00 2d e6 07 0d 00 00 00 01 03 a0 00 03 e0 00 00 .-..............
- 070: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 0a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 0b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 0d0: 00 00 00 00 00 c1 00 00 00 00 00 00 00 00 00 00 ................
- 0e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 150: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 160: 00 83 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 170: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 180: 00 00 00 00 00 00 00 00 00 00 07 00 30 00 00 00 ............0...
- 190: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 1c0: 02 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 ................
- 1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 1f0: 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 220: 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 230: 0c 00 00 00 00 00 00 60 00 00 00 06 00 00 c3 00 .......`........
- 240: 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 270: 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 290: 04 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 2b0: 00 00 00 00 83 00 8c 00 00 00 00 00 00 00 00 00 ................
- 2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 310: 00 78 00 00 00 00 00 00 00 00 00 00 00 00 70 00 .x............p.
- 320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 340: 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 350: 00 00 00 00 00 68 00 00 00 00 00 00 00 00 00 00 .....h..........
- 360: 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 ................
- 370: 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 ................
- 380: 00 00 00 00 70 00 00 00 00 00 00 00 00 00 00 00 ....p...........
- 390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
- 3a0: 5e 01 07 17 1b 1b 01 81 13 74 61 62 6c 65 73 65 ^........tablese
- 3b0: 6e 73 6f 32 73 73 65 6e 73 6f 72 73 02 43 52 45 nso2ssensors.CRE
- 3c0: 41 54 45 20 54 41 42 4c 45 20 73 65 6e 73 6f 72 ATE TABLE sensor
- 3d0: 73 20 0a 20 20 24 20 20 20 20 20 20 20 20 20 20 s .  $          
- 3e0: b8 6e 61 6d 65 21 74 65 78 74 2c 20 79 61 6c 20 .name!text, yal 
- 3f0: 72 65 61 6c 2c 20 74 69 6d 65 20 74 65 78 74 29 real, time text)
-  ') WHERE pgno=1
-}
-
-db close
-sqlite3 db test.db
-
-do_catchsql_test 3.3 {
-  PRAGMA integrity_check;
-} {1 {database disk image is malformed}}
+} ;# ifcapable vtab
 
 
 

From 063cf17a113a79b503d8dc6df7989414b6804a83 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sun, 22 Oct 2017 08:02:56 +0000
Subject: [PATCH 201/270] Fix test cases in dbpage.test so that they work with
 SQLITE_DEFAULT_AUTOVACUUM=1 builds.

FossilOrigin-Name: 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/dbpage.test |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 07fa1c88b3..68f0dd8fc9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\srunning\stests\sthat\suse\ssqlite_dbpage\swith\sSQLITE_OMIT_VIRTUAL_TABLE\nbuilds.
-D 2017-10-22T07:57:29.038
+C Fix\stest\scases\sin\sdbpage.test\sso\sthat\sthey\swork\swith\nSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds.
+D 2017-10-22T08:02:56.288
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -713,7 +713,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b
 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
-F test/dbpage.test 9cf4dc92a4de67c81e5c32b24e3fbb8c4757e4b642694a219b3090a4f9277a4d
+F test/dbpage.test a9e7774416f5f2fb237601628804b339f897112edb9ba92f7675a61e3e9c6244
 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ee8cb6ae5fd076ec226bb184b5690ba29f9df8cfaef47aaf13336873b4c1f6c
-R 2ad0f857ce1c547a47a99955708ad02e
+P 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528
+R 6416cb828bdd2d9e1b57b650566d1c6c
 U dan
-Z 31291a1d9ccb37db331d41df009b953d
+Z 9dddfd6ebe074eda07e6c2b88ec3bea0
diff --git a/manifest.uuid b/manifest.uuid
index 321fef9696..e9c39dc6ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528
\ No newline at end of file
+0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7
\ No newline at end of file
diff --git a/test/dbpage.test b/test/dbpage.test
index e29d4b33a9..e4389a3b90 100644
--- a/test/dbpage.test
+++ b/test/dbpage.test
@@ -22,6 +22,7 @@ ifcapable !vtab||!compound {
 }
 
 do_execsql_test 100 {
+  PRAGMA auto_vacuum=0;
   PRAGMA page_size=4096;
   PRAGMA journal_mode=WAL;
   CREATE TABLE t1(a,b);

From f970372698fc57380c8756b3809ed13ca246dcec Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Oct 2017 07:52:07 +0000
Subject: [PATCH 202/270] Fix minor test script problems revealed by
 releasetest.tcl.

FossilOrigin-Name: 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
---
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/corruptK.test |  2 ++
 test/dbpage.test   | 22 +++++++++++++---------
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 68f0dd8fc9..994305976a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stest\scases\sin\sdbpage.test\sso\sthat\sthey\swork\swith\nSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds.
-D 2017-10-22T08:02:56.288
+C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl.
+D 2017-10-23T07:52:07.660
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -690,7 +690,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51
 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454
 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4
 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
-F test/corruptK.test d93f5797bf0555a47dcc81bc9727cb72fa7c8ac348c25f42937573cd29bb3850
+F test/corruptK.test 91550557849244a9904f4e090052e3f2c1c3f1106840d58b00ffaa3a8c2d3fc0
 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8
 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c
 F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c
@@ -713,7 +713,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b
 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
-F test/dbpage.test a9e7774416f5f2fb237601628804b339f897112edb9ba92f7675a61e3e9c6244
+F test/dbpage.test 10b9e91d07b0892444fff4578706648e955b5fb260218298f838da74f0d9d211
 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7bd20a20a0f422765a6e469d1a1b67b95c4a441931688e04a2b1b491a5238528
-R 6416cb828bdd2d9e1b57b650566d1c6c
+P 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7
+R e3bec412c3fcc940b80e7c9735d18bbe
 U dan
-Z 9dddfd6ebe074eda07e6c2b88ec3bea0
+Z a4aaac778f287d83315a30f9e1092784
diff --git a/manifest.uuid b/manifest.uuid
index e9c39dc6ef..be55c08755 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7
\ No newline at end of file
+15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
\ No newline at end of file
diff --git a/test/corruptK.test b/test/corruptK.test
index 05465a9b5f..1694db1747 100644
--- a/test/corruptK.test
+++ b/test/corruptK.test
@@ -110,6 +110,7 @@ do_catchsql_test 2.3 {
 #-------------------------------------------------------------------------
 
 ifcapable vtab {
+if {[permutation]!="inmemory_journal"} {
 
   proc hex2blob {hex} {
     # Split on newlines:
@@ -219,6 +220,7 @@ ifcapable vtab {
     PRAGMA integrity_check;
   } {1 {database disk image is malformed}}
 
+} ;# [permutation]!="inmemory_journal"
 } ;# ifcapable vtab
 
 
diff --git a/test/dbpage.test b/test/dbpage.test
index e4389a3b90..a8ef8c0800 100644
--- a/test/dbpage.test
+++ b/test/dbpage.test
@@ -21,15 +21,19 @@ ifcapable !vtab||!compound {
   return
 }
 
-do_execsql_test 100 {
-  PRAGMA auto_vacuum=0;
-  PRAGMA page_size=4096;
-  PRAGMA journal_mode=WAL;
-  CREATE TABLE t1(a,b);
-  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
-  INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c;
-  PRAGMA integrity_check;
-} {wal ok}
+do_test 100 {
+  execsql {
+    PRAGMA auto_vacuum=0;
+    PRAGMA page_size=4096;
+    PRAGMA journal_mode=WAL;
+  }
+  execsql { 
+    CREATE TABLE t1(a,b);
+    WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+      INSERT INTO t1(a,b) SELECT x, printf('%d-x%.*c',x,x,'x') FROM c;
+    PRAGMA integrity_check;
+  }
+} {ok}
 do_execsql_test 110 {
   SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('main') ORDER BY pgno;
 } {1 X'53514C6974' 2 X'0500000001' 3 X'0D0000004E' 4 X'0D00000016'}

From b5ef728db66284c47861ea66a7e906a9a4b33cb4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Oct 2017 11:48:00 +0000
Subject: [PATCH 203/270] In lsm (not SQLite), avoid opening the *-shm file
 before the database file is locked. Otherwise, there is a race condition
 whereby a connection in another process process may unlink() it while it is
 still in use.

FossilOrigin-Name: c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad
---
 ext/lsm1/lsm_shared.c | 10 ++++------
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c
index 83e44b4705..c4bb9d6a23 100644
--- a/ext/lsm1/lsm_shared.c
+++ b/ext/lsm1/lsm_shared.c
@@ -340,9 +340,6 @@ static int doDbConnect(lsm_db *pDb){
   /* Obtain a pointer to the shared-memory header */
   assert( pDb->pShmhdr==0 );
   assert( pDb->bReadonly==0 );
-  rc = lsmShmCacheChunks(pDb, 1);
-  if( rc!=LSM_OK ) return rc;
-  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Block for an exclusive lock on DMS1. This lock serializes all calls
   ** to doDbConnect() and doDbDisconnect() across all processes.  */
@@ -353,10 +350,11 @@ static int doDbConnect(lsm_db *pDb){
     nUs = nUs * 2;
     if( nUs>nUsMax ) nUs = nUsMax;
   }
-  if( rc!=LSM_OK ){
-    pDb->pShmhdr = 0;
-    return rc;
+  if( rc==LSM_OK ){
+    rc = lsmShmCacheChunks(pDb, 1);
   }
+  if( rc!=LSM_OK ) return rc;
+  pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
 
   /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first 
   ** and only connection to the database. In this case initialize the 
diff --git a/manifest b/manifest
index 994305976a..3bd79dea76 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl.
-D 2017-10-23T07:52:07.660
+C In\slsm\s(not\sSQLite),\savoid\sopening\sthe\s*-shm\sfile\sbefore\sthe\sdatabase\sfile\sis\nlocked.\sOtherwise,\sthere\sis\sa\srace\scondition\swhereby\sa\sconnection\sin\sanother\nprocess\sprocess\smay\sunlink()\sit\swhile\sit\sis\sstill\sin\suse.
+D 2017-10-23T11:48:00.248
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -245,7 +245,7 @@ F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709
 F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
-F ext/lsm1/lsm_shared.c 5bc37768e558492f60d7196735ddd54843cd239bd66c1af6eb205a6348ca5e46
+F ext/lsm1/lsm_shared.c 1a76b7a5e89a003c24d58f1fb295c4203b48ef6acba9a194ac6003ade09fcd47
 F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0fcf1e61ef2b6c1fb8f4846f973d2d427598ffbc80dbc49e56e71513b194bae7
-R e3bec412c3fcc940b80e7c9735d18bbe
+P 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
+R 788a72b4e4bc49507efed4d645930eed
 U dan
-Z a4aaac778f287d83315a30f9e1092784
+Z 97cc68bc0ee206aadd31e81854de499f
diff --git a/manifest.uuid b/manifest.uuid
index be55c08755..abddaa098d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
\ No newline at end of file
+c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad
\ No newline at end of file

From 21cd29ab15630e29dc75dadbd8bb8d1e8fffbfc7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Oct 2017 16:03:54 +0000
Subject: [PATCH 204/270] Avoid passing a NULL value as the second argument to
 memcpy().

FossilOrigin-Name: 58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/expr.c    |  2 +-
 src/vdbe.c    | 11 +++--------
 4 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 5ff2166ddf..5af1c47e9c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sminor\stest\sscript\sproblems\srevealed\sby\sreleasetest.tcl.
-D 2017-10-23T07:53:00.457
+C Avoid\spassing\sa\sNULL\svalue\sas\sthe\ssecond\sargument\sto\smemcpy().
+D 2017-10-23T16:03:54.957
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -414,7 +414,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a
+F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -530,7 +530,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
+F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9474f061d634721ca462bbad375270b8b39dea44011df3e1aac72021e8bf70fc 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
-R fca4c2998d860b91137d3cb7520f7137
+P c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831
+R 676a97200c74a615c9238afe630ac524
 U dan
-Z 456ef7edc59a34c970d90601a596e58f
+Z 81b9a8d334381cc6ba3e8852cc37f621
diff --git a/manifest.uuid b/manifest.uuid
index 97cae8c731..7a241c2c14 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831
\ No newline at end of file
+58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 0b922cd0fd..31e9a93865 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -2678,7 +2678,7 @@ int sqlite3CodeSubselect(
         /* Loop through each expression in <exprlist>. */
         r1 = sqlite3GetTempReg(pParse);
         r2 = sqlite3GetTempReg(pParse);
-        if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2);
+        if( isRowid ) sqlite3VdbeAddOp4(v, OP_Blob, 0, r2, 0, "", P4_STATIC);
         for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){
           Expr *pE2 = pItem->pExpr;
           int iValToIns;
diff --git a/src/vdbe.c b/src/vdbe.c
index 9687170bec..cfe18a9d1e 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4435,14 +4435,9 @@ case OP_InsertInt: {
 
   if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
   if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey;
-  if( pData->flags & MEM_Null ){
-    x.pData = 0;
-    x.nData = 0;
-  }else{
-    assert( pData->flags & (MEM_Blob|MEM_Str) );
-    x.pData = pData->z;
-    x.nData = pData->n;
-  }
+  assert( pData->flags & (MEM_Blob|MEM_Str) );
+  x.pData = pData->z;
+  x.nData = pData->n;
   seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
   if( pData->flags & MEM_Zero ){
     x.nZero = pData->u.nZero;

From efeaec365f44f69576158411bd7f7a27304b7e64 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 23 Oct 2017 16:34:07 +0000
Subject: [PATCH 205/270] Updates to test procedures so that they all work with
 encryption.  No changes to the core SQLite code.

FossilOrigin-Name: 0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd
---
 manifest             | 22 +++++++++++-----------
 manifest.uuid        |  2 +-
 src/test_config.c    |  6 ++++++
 test/pragma.test     | 43 ++++++++++++++++++++++---------------------
 test/schema6.test    |  1 +
 test/swarmvtab.test  |  2 +-
 test/swarmvtab2.test |  1 +
 7 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/manifest b/manifest
index 5af1c47e9c..4bf6a1d006 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\spassing\sa\sNULL\svalue\sas\sthe\ssecond\sargument\sto\smemcpy().
-D 2017-10-23T16:03:54.957
+C Updates\sto\stest\sprocedures\sso\sthat\sthey\sall\swork\swith\sencryption.\s\sNo\schanges\nto\sthe\score\sSQLite\scode.
+D 2017-10-23T16:34:07.586
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -486,7 +486,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c abf6fc1fe9d041b699578c42e3db81f8831c4f5b804f1927958102ee8f2b773e
+F src/test_config.c 3000f00b9b47b149d842059904c3fcab5f3871fb6aee7d7cc5756f0c64779ae3
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -1106,7 +1106,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
 F test/permutations.test 490e3333b9b1aefb7ebc6e9ab2ae0e382b7dd8713ccc4a2786b0f75467c2ab6b
-F test/pragma.test c31b5e98998c160a4c85b1e04f590655c67f2daa7f73854640cd120610e3ac15
+F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b335c
@@ -1163,7 +1163,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schema6.test 5b21bbdd405bc93b3e6af5e6ece64d230e35f65cc4035e5c2b89fc8a090d7270
+F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce1f9b
 F test/securedel.test 5f997cb6bd38727b81e0985f53ec386c99db6441b2b9e6357240649d29017239
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7
@@ -1249,8 +1249,8 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
 F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
-F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17
-F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d
+F test/swarmvtab.test c2279311b44de032f86a8295a9b06818d864856f9428b4c99eee91a0d419cf25
+F test/swarmvtab2.test 9a3a68a1e58d00f4ed6c68d12d52f2df971b9e22a80a41f6f8c1409abba8e5b4
 F test/swarmvtabfault.test 73563eefe3073c6fb3bb14475fb4ef5d4f2e3a67a02947ee0ca08980ea3dd7fe
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c333ee7e9677bda1dad8e80cf94755350bfe20bc0754577442727b6f807ad831
-R 676a97200c74a615c9238afe630ac524
-U dan
-Z 81b9a8d334381cc6ba3e8852cc37f621
+P 58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577
+R a447261df1989a937247d3824cdf45d6
+U drh
+Z c50c87663ba16acea2ad67bd15f69a8a
diff --git a/manifest.uuid b/manifest.uuid
index 7a241c2c14..3a2fc73c3f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-58e5bb491bc0e28412ca188ba8fbe94adc67b5b1e9d7c112c36011400528d577
\ No newline at end of file
+0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd
\ No newline at end of file
diff --git a/src/test_config.c b/src/test_config.c
index 9ff84dcec0..193f95edf9 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -214,6 +214,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "json1", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_HAS_CODEC
+  Tcl_SetVar2(interp, "sqlite_options", "has_codec", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "has_codec", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS
   Tcl_SetVar2(interp, "sqlite_options", "like_match_blobs", "0", TCL_GLOBAL_ONLY);
 #else
diff --git a/test/pragma.test b/test/pragma.test
index e4c79111ba..90cd7e2dfe 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -1934,26 +1934,27 @@ do_test 23.5 {
 } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE}
 db2 close
 
-reset_db
-do_execsql_test 24.0 {
-  PRAGMA page_size = 1024;
-  CREATE TABLE t1(a, b, c);
-  CREATE INDEX i1 ON t1(b);
-  INSERT INTO t1 VALUES('a', 'b', 'c');
-  PRAGMA integrity_check;
-} {ok}
-
-set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}]
-db close
-hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263
-
-sqlite3 db test.db
-do_catchsql_test 24.1 {
-  SELECT * FROM t1;
-} {1 {database disk image is malformed}}
-do_catchsql_test 24.2 {
-  PRAGMA integrity_check;
-} {0 {{database disk image is malformed}}}
-
+ifcapable !has_codec {
+  reset_db
+  do_execsql_test 24.0 {
+    PRAGMA page_size = 1024;
+    CREATE TABLE t1(a, b, c);
+    CREATE INDEX i1 ON t1(b);
+    INSERT INTO t1 VALUES('a', 'b', 'c');
+    PRAGMA integrity_check;
+  } {ok}
+  
+  set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}]
+  db close
+  hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263
+  
+  sqlite3 db test.db
+  do_catchsql_test 24.1 {
+    SELECT * FROM t1;
+  } {1 {database disk image is malformed}}
+  do_catchsql_test 24.2 {
+    PRAGMA integrity_check;
+  } {0 {{database disk image is malformed}}}
+}  
 database_never_corrupt
 finish_test
diff --git a/test/schema6.test b/test/schema6.test
index 7de04d51c2..0eb8a1c434 100644
--- a/test/schema6.test
+++ b/test/schema6.test
@@ -18,6 +18,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set ::testprefix schema6
+do_not_use_codec
 
 # Command:   check_same_database_content TESTNAME SQL1 SQL2 SQL3 ...
 #
diff --git a/test/swarmvtab.test b/test/swarmvtab.test
index c8128ef181..4cdcf29ca8 100644
--- a/test/swarmvtab.test
+++ b/test/swarmvtab.test
@@ -15,6 +15,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix swarmvtab
+do_not_use_codec
 
 ifcapable !vtab {
   finish_test
@@ -243,4 +244,3 @@ do_catchsql_test 3.3.2 { SELECT * FROM xyz } {1 {fetch_db error!}}
 
 
 finish_test
-
diff --git a/test/swarmvtab2.test b/test/swarmvtab2.test
index 2b37fd2ac8..cf1bdd0af9 100644
--- a/test/swarmvtab2.test
+++ b/test/swarmvtab2.test
@@ -15,6 +15,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix swarmvtab
+do_not_use_codec
 
 ifcapable !vtab {
   finish_test

From 15dd780a9eb97f31368410299549f5ca0b70491b Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 23 Oct 2017 17:36:52 +0000
Subject: [PATCH 206/270] Cleanup a test database file prior to using it.

FossilOrigin-Name: e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b
---
 manifest                 | 14 +++++++-------
 manifest.uuid            |  2 +-
 test/swarmvtabfault.test |  2 ++
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 3bd79dea76..336384dfde 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\slsm\s(not\sSQLite),\savoid\sopening\sthe\s*-shm\sfile\sbefore\sthe\sdatabase\sfile\sis\nlocked.\sOtherwise,\sthere\sis\sa\srace\scondition\swhereby\sa\sconnection\sin\sanother\nprocess\sprocess\smay\sunlink()\sit\swhile\sit\sis\sstill\sin\suse.
-D 2017-10-23T11:48:00.248
+C Cleanup\sa\stest\sdatabase\sfile\sprior\sto\susing\sit.
+D 2017-10-23T17:36:52.561
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -1251,7 +1251,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
 F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17
 F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d
-F test/swarmvtabfault.test 73563eefe3073c6fb3bb14475fb4ef5d4f2e3a67a02947ee0ca08980ea3dd7fe
+F test/swarmvtabfault.test f8ae84cd44454555386c393892280f85f5ab35dcd0797641830fd7ebbfcc9d15
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 15e87d9bc81812fb9de86f9870389217dddb9f9c4d48e42acefce2d35ba293f1
-R 788a72b4e4bc49507efed4d645930eed
-U dan
-Z 97cc68bc0ee206aadd31e81854de499f
+P c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad
+R 8a4b9bbdc7dfd424f00b7f6b2d5817a7
+U mistachkin
+Z b3872513ac1c114df2a01eba444b5f49
diff --git a/manifest.uuid b/manifest.uuid
index abddaa098d..3ab3d6091f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad
\ No newline at end of file
+e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b
\ No newline at end of file
diff --git a/test/swarmvtabfault.test b/test/swarmvtabfault.test
index 8c913c1066..dfdd16705b 100644
--- a/test/swarmvtabfault.test
+++ b/test/swarmvtabfault.test
@@ -29,6 +29,8 @@ proc fetch_db {file} {
 }
 
 forcedelete test.db1
+forcedelete test.db2
+
 do_execsql_test 1.0 {
   ATTACH 'test.db1' AS aux;
   CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b);

From 0295774ca0d2d33c7644688be3624433e29ab84b Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 23 Oct 2017 20:17:19 +0000
Subject: [PATCH 207/270] In the 'swarmvtabfault' test module, make sure to
 close the database handle prior to raising an error from the 'not found'
 callback script.

FossilOrigin-Name: f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b
---
 manifest                 | 12 ++++++------
 manifest.uuid            |  2 +-
 test/swarmvtabfault.test |  5 ++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 336384dfde..2cb9e12473 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cleanup\sa\stest\sdatabase\sfile\sprior\sto\susing\sit.
-D 2017-10-23T17:36:52.561
+C In\sthe\s'swarmvtabfault'\stest\smodule,\smake\ssure\sto\sclose\sthe\sdatabase\shandle\sprior\sto\sraising\san\serror\sfrom\sthe\s'not\sfound'\scallback\sscript.
+D 2017-10-23T20:17:19.102
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -1251,7 +1251,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
 F test/swarmvtab.test 05c4ca7b6ab0cc6f4c335a510347f99d741fa71366004699cf7dfa3cff4e2d17
 F test/swarmvtab2.test 038ef9bcad6fd2fb9e395196080cf23e223ddb1219015049a61540c161bc577d
-F test/swarmvtabfault.test f8ae84cd44454555386c393892280f85f5ab35dcd0797641830fd7ebbfcc9d15
+F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
 F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c8b8b9b573665c2b4d278bb679a48ac407374b8d860f0c3f4d50a303043464ad
-R 8a4b9bbdc7dfd424f00b7f6b2d5817a7
+P e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b
+R c767dcd50a85f97407be83eb8fcdc039
 U mistachkin
-Z b3872513ac1c114df2a01eba444b5f49
+Z 6d76c482979023bebbeef80b070aef99
diff --git a/manifest.uuid b/manifest.uuid
index 3ab3d6091f..0c9cd50350 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e1bf5e1d06dbe573671c6b5472b45f5bfa3cf82374f70a76b4c80815bf15a29b
\ No newline at end of file
+f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b
\ No newline at end of file
diff --git a/test/swarmvtabfault.test b/test/swarmvtabfault.test
index dfdd16705b..083d80df2e 100644
--- a/test/swarmvtabfault.test
+++ b/test/swarmvtabfault.test
@@ -24,8 +24,11 @@ ifcapable !vtab {
 proc fetch_db {file} {
   forcedelete $file
   sqlite3 dbX $file
-  dbX eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b) }
+  set rc [catch {
+    dbX eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b) }
+  } res]
   dbX close
+  if {$rc!=0} {error $res}
 }
 
 forcedelete test.db1

From f7902a543eceb3dbae5d9fea51bf83ed16a8d4c6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 23 Oct 2017 21:24:38 +0000
Subject: [PATCH 208/270] On the amalgamation package, enable the sqlite_dbpage
 virtual table so that the .dbinfo command works in the CLI.

FossilOrigin-Name: 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6
---
 autoconf/Makefile.am |  2 +-
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am
index e8211596d9..8c046f822d 100644
--- a/autoconf/Makefile.am
+++ b/autoconf/Makefile.am
@@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 
 include_HEADERS = sqlite3.h sqlite3ext.h
 
diff --git a/manifest b/manifest
index 2399bd7eea..ce4682de02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\sfixes\sto\sswarmvtabfault.test\sand\sthe\slsm\sextension\sinto\sthis\nbranch.\sNo\schanges\sto\score\sSQLite\scode.
-D 2017-10-23T20:20:42.832
+C On\sthe\samalgamation\spackage,\senable\sthe\ssqlite_dbpage\svirtual\stable\sso\sthat\nthe\s.dbinfo\scommand\sworks\sin\sthe\sCLI.
+D 2017-10-23T21:24:38.787
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -10,7 +10,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
-F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4
+F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
 F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0481330756e861de8e1eb7136e7dca2f333b2a82377949d2e0eba401af7d73dd f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b
-R ddc3003da6d6bb8e92c74828b5fa4343
-U dan
-Z d8722be9be441804446b8c51f09c2f27
+P f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77
+R f52ceae69cf9eb99d83d95eeda09f025
+U drh
+Z 919379e1b338add2dc31d98f7ca8060a
diff --git a/manifest.uuid b/manifest.uuid
index 1fd8f6b26b..5ff9088210 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77
\ No newline at end of file
+54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6
\ No newline at end of file

From 952523f618d75ffe9e085b65cab63f53684c464a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 24 Oct 2017 17:28:25 +0000
Subject: [PATCH 209/270] Fix a problem in the incr-blob code causing a new
 cursor to be opened for every sqlite3_blob_reopen() call.

FossilOrigin-Name: 41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/vdbeblob.c |  7 ++++---
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index ce4682de02..3ca17dd781 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C On\sthe\samalgamation\spackage,\senable\sthe\ssqlite_dbpage\svirtual\stable\sso\sthat\nthe\s.dbinfo\scommand\sworks\sin\sthe\sCLI.
-D 2017-10-23T21:24:38.787
+C Fix\sa\sproblem\sin\sthe\sincr-blob\scode\scausing\sa\snew\scursor\sto\sbe\sopened\sfor\nevery\ssqlite3_blob_reopen()\scall.
+D 2017-10-24T17:28:25.843
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -535,7 +535,7 @@ F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c c423065d50cee24bc8cba57764f5e9869a1bb920c50907f5dd363ebd7c5aef82
-F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
+F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f632b87002a7ee9be569aacb5ce85bb7e9c86b7d77bb8d1bf8b55eb5be4ecc77
-R f52ceae69cf9eb99d83d95eeda09f025
-U drh
-Z 919379e1b338add2dc31d98f7ca8060a
+P 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6
+R ba55e91a353da744095752f95418b3a4
+U dan
+Z 67fa09a204d6f78af71556879bc0e0f6
diff --git a/manifest.uuid b/manifest.uuid
index 5ff9088210..7c05a49d6e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6
\ No newline at end of file
+41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c
\ No newline at end of file
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index 16a2c0f36f..4279792696 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -63,11 +63,12 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
   v->aMem[1].u.i = iRow;
 
   /* If the statement has been run before (and is paused at the OP_ResultRow)
-  ** then back it up to the point where it does the OP_SeekRowid.  This could
+  ** then back it up to the point where it does the OP_NotExists.  This could
   ** have been down with an extra OP_Goto, but simply setting the program
   ** counter is faster. */
-  if( v->pc>3 ){
-    v->pc = 3;
+  if( v->pc>4 ){
+    v->pc = 4;
+    assert( v->aOp[v->pc].opcode==OP_NotExists );
     rc = sqlite3VdbeExec(v);
   }else{
     rc = sqlite3_step(p->pStmt);

From 39f2b329f6513b6901afb461b57674a109e934ff Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 24 Oct 2017 18:55:49 +0000
Subject: [PATCH 210/270] Version 3.21.0

FossilOrigin-Name: 1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827
---
 manifest      | 13 ++++++++-----
 manifest.uuid |  2 +-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/manifest b/manifest
index 3ca17dd781..1a60fff9b1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\sincr-blob\scode\scausing\sa\snew\scursor\sto\sbe\sopened\sfor\nevery\ssqlite3_blob_reopen()\scall.
-D 2017-10-24T17:28:25.843
+C Version\s3.21.0
+D 2017-10-24T18:55:49.833
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -1664,7 +1664,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 54b90b4f27f6f213825e881929fa04a6282e5b909a12e881366f9aef6872dcb6
+P 41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c
 R ba55e91a353da744095752f95418b3a4
-U dan
-Z 67fa09a204d6f78af71556879bc0e0f6
+T +bgcolor * #d0c0ff
+T +sym-release *
+T +sym-version-3.21.0 *
+U drh
+Z c165baaf48a254c276ef91f282f216a1
diff --git a/manifest.uuid b/manifest.uuid
index 7c05a49d6e..b772210fa4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-41ef34a1f0650c50cacb203665cd9d57db53a49c979bf0d5a78937517f763a2c
\ No newline at end of file
+1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827
\ No newline at end of file

From ea78f64dc11da4468621bee30a8b6f01fc1ba25e Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 24 Oct 2017 21:17:12 +0000
Subject: [PATCH 211/270] Force the autoconf makefile for MSVC to always use
 the 'find.exe' included with Windows.

FossilOrigin-Name: b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887
---
 autoconf/Makefile.msc |  2 +-
 manifest              | 16 ++++++++--------
 manifest.uuid         |  2 +-
 tool/mkmsvcmin.tcl    |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index e0f7ad8615..39831da03e 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -974,7 +974,7 @@ sqlite3.lo:	$(SQLITE3C)
 !IF $(USE_RC)!=0
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
     ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF
diff --git a/manifest b/manifest
index 918de05b01..d4ab8e50ee 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sall\sof\sthe\sfixes\sfrom\sthe\s3.21.0\srelease.
-D 2017-10-24T19:05:26.063
+C Force\sthe\sautoconf\smakefile\sfor\sMSVC\sto\salways\suse\sthe\s'find.exe'\sincluded\swith\sWindows.
+D 2017-10-24T21:17:12.950
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
-F autoconf/Makefile.msc 645b8a9774281515dc4a8de65d8a914f7b418ba8bd1c48b53ccbf43d3b339715
+F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578
@@ -1596,7 +1596,7 @@ F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c7
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
 F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c
-F tool/mkmsvcmin.tcl cbd93f1cfa3a0a9ae56fc958510aa3fc3ac65e29cb111716199e3d0e66eefaa4
+F tool/mkmsvcmin.tcl 8baf26690b80d861d0ac341b29880eec6ade39e4f11fe690271ded9cb90563a3
 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
@@ -1664,7 +1664,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f317037b31b19007e30bb41c54506d9d241a6d590c2b31e2ddda10d4f6e7605b 1a584e499906b5c87ec7d43d4abce641fdf017c42125b083109bc77c4de48827
-R ba55e91a353da744095752f95418b3a4
-U drh
-Z c9856165fc8b1b55de6e521677f40ace
+P 5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441
+R 54e49932e96b845a01a6eb1a3cc08724
+U mistachkin
+Z 580a702e0e85c423f77f3c922262f566
diff --git a/manifest.uuid b/manifest.uuid
index 9810bd8910..5f2c913791 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441
\ No newline at end of file
+b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887
\ No newline at end of file
diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl
index 8d5729865c..1d646ef9f5 100644
--- a/tool/mkmsvcmin.tcl
+++ b/tool/mkmsvcmin.tcl
@@ -54,7 +54,7 @@ proc substVars { data } {
 set blocks(1) [string trimleft [string map [list \\\\ \\] {
 _HASHCHAR=^#
 !IF ![echo !IFNDEF VERSION > rcver.vc] && \\
-    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\
+    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \\
     ![echo !ENDIF >> rcver.vc]
 !INCLUDE rcver.vc
 !ENDIF

From 98d529b5411816e75a038f10e4666c20a42a7628 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 25 Oct 2017 01:34:24 +0000
Subject: [PATCH 212/270] Add the sqlite_btreeinfo eponymous-only table for
 introspecting the schema and estimating the sizes of various btrees.

FossilOrigin-Name: 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d
---
 ext/misc/btreeinfo.c | 425 +++++++++++++++++++++++++++++++++++++++++++
 manifest             |  13 +-
 manifest.uuid        |   2 +-
 3 files changed, 433 insertions(+), 7 deletions(-)
 create mode 100644 ext/misc/btreeinfo.c

diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c
new file mode 100644
index 0000000000..131c63e835
--- /dev/null
+++ b/ext/misc/btreeinfo.c
@@ -0,0 +1,425 @@
+/*
+** 2017-10-24
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of the "sqlite_btreeinfo" virtual table.
+**
+** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual
+** table that shows information about all btrees in an SQLite database file.
+** The schema is like this:
+**
+** CREATE TABLE sqlite_btreeinfo(
+**    type TEXT,                   -- "table" or "index"
+**    name TEXT,                   -- Name of table or index for this btree.
+**    tbl_name TEXT,               -- Associated table
+**    rootpage INT,                -- The root page of the btree
+**    sql TEXT,                    -- SQL for this btree - from sqlite_master
+**    hasRowid BOOLEAN,            -- True if the btree has a rowid
+**    nEntry INT,                  -- Estimated number of enteries
+**    nPage INT,                   -- Estimated number of pages
+**    depth INT,                   -- Depth of the btree
+**    szPage INT,                  -- Size of each page in bytes
+**    zSchema TEXT HIDDEN          -- The schema to which this btree belongs
+** );
+**
+** The first 5 fields are taken directly from the sqlite_master table.
+** Considering only the first 5 fields, the only difference between 
+** this virtual table and the sqlite_master table is that this virtual
+** table omits all entries that have a 0 or NULL rowid - in other words
+** it omits triggers and views.
+**
+** The value added by this table comes in the next 5 fields.
+**
+** Note that nEntry and nPage are *estimated*.  They are computed doing
+** a single search from the root to a leaf, counting the number of cells
+** at each level, and assuming that unvisited pages have a similar number
+** of cells.
+**
+** The sqlite_dbpage virtual table must be available for this virtual table
+** to operate.
+**
+** USAGE EXAMPLES:
+**
+** Show the table btrees in a schema order with the tables with the most
+** rows occuring first:
+**
+**      SELECT name, nEntry
+**        FROM sqlite_btreeinfo
+**       WHERE type='table'
+**       ORDER BY nEntry DESC, name;
+**
+** Show the names of all WITHOUT ROWID tables: 
+**
+**      SELECT name FROM sqlite_btreeinfo
+**       WHERE type='table' AND NOT hasRowid;
+*/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* Columns available in this virtual table */
+#define BINFO_COLUMN_TYPE         0
+#define BINFO_COLUMN_NAME         1
+#define BINFO_COLUMN_TBL_NAME     2
+#define BINFO_COLUMN_ROOTPAGE     3
+#define BINFO_COLUMN_SQL          4
+#define BINFO_COLUMN_HASROWID     5
+#define BINFO_COLUMN_NENTRY       6
+#define BINFO_COLUMN_NPAGE        7
+#define BINFO_COLUMN_DEPTH        8
+#define BINFO_COLUMN_SZPAGE       9
+#define BINFO_COLUMN_SCHEMA      10
+
+/* Forward declarations */
+typedef struct BinfoTable BinfoTable;
+typedef struct BinfoCursor BinfoCursor;
+
+/* A cursor for the sqlite_btreeinfo table */
+struct BinfoCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_stmt *pStmt;            /* Query against sqlite_master */
+  int rc;                         /* Result of previous sqlite_step() call */
+  int hasRowid;                   /* hasRowid value.  Negative if unknown. */
+  sqlite3_int64 nEntry;           /* nEntry value */
+  int nPage;                      /* nPage value */
+  int depth;                      /* depth value */
+  int szPage;                     /* size of a btree page.  0 if unknown */
+  char *zSchema;                  /* Schema being interrogated */
+};
+
+/* The sqlite_btreeinfo table */
+struct BinfoTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;                    /* The databse connection */
+};
+
+/*
+** Connect to the sqlite_btreeinfo virtual table.
+*/
+static int binfoConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  BinfoTable *pTab = 0;
+  int rc = SQLITE_OK;
+  rc = sqlite3_declare_vtab(db, 
+      "CREATE TABLE x(\n"
+      " type TEXT,\n"
+      " name TEXT,\n"
+      " tbl_name TEXT,\n"
+      " rootpage INT,\n"
+      " sql TEXT,\n"
+      " hasRowid BOOLEAN,\n"
+      " nEntry INT,\n"
+      " nPage INT,\n"
+      " depth INT,\n"
+      " szPage INT,\n"
+      " zSchema TEXT HIDDEN\n"
+      ")");
+  if( rc==SQLITE_OK ){
+    pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM;
+  }
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( pTab ){
+    pTab->db = db;
+  }
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a btreeinfo virtual table.
+*/
+static int binfoDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** idxNum:
+**
+**     0     Use "main" for the schema
+**     1     Schema identified by parameter ?1
+*/
+static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
+  pIdxInfo->estimatedCost = 10000.0;  /* Cost estimate */
+  pIdxInfo->estimatedRows = 100;
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->usable
+     && p->iColumn==BINFO_COLUMN_SCHEMA
+     && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+    ){
+      pIdxInfo->estimatedCost = 1000.0;
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].omit = 1;
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  BinfoCursor *pCsr;
+
+  pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(BinfoCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int binfoClose(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr->zSchema);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int binfoNext(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  pCsr->rc = sqlite3_step(pCsr->pStmt);
+  pCsr->hasRowid = -1;
+  return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int binfoEof(sqlite3_vtab_cursor *pCursor){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  return pCsr->rc!=SQLITE_ROW;
+}
+
+/* Position a cursor back to the beginning.
+*/
+static int binfoFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  BinfoTable *pTab = (BinfoTable *)pCursor->pVtab;
+  char *zSql;
+  int rc;
+
+  sqlite3_free(pCsr->zSchema);
+  if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){
+    pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+  }else{
+    pCsr->zSchema = sqlite3_mprintf("main");
+  }
+  zSql = sqlite3_mprintf(
+      "SELECT 0, 'table','sqlite_master','sqlite_master',1,NULL "
+      "UNION ALL "
+      "SELECT rowid, type, name, tbl_name, rootpage, sql"
+      " FROM \"%w\".sqlite_master WHERE rootpage>=1",
+       pCsr->zSchema);
+  sqlite3_finalize(pCsr->pStmt);
+  pCsr->pStmt = 0;
+  pCsr->hasRowid = -1;
+  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc==SQLITE_OK ){
+    rc = binfoNext(pCursor);
+  }
+  return rc;
+}
+
+/* Decode big-endian integers */
+static unsigned int get_uint16(unsigned char *a){
+  return (a[0]<<8)|a[1];
+}
+static unsigned int get_uint32(unsigned char *a){
+  return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3];
+}
+
+/* Examine the b-tree rooted at pgno and estimate its size.
+** Return non-zero if anything goes wrong.
+*/
+static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){
+  sqlite3_int64 nEntry = 1;
+  int nPage = 1;
+  unsigned char *aData;
+  sqlite3_stmt *pStmt = 0;
+  int rc = SQLITE_OK;
+  int pgsz = 0;
+  int nCell;
+  int iCell;
+
+  rc = sqlite3_prepare_v2(db, 
+           "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1,
+           &pStmt, 0);
+  if( rc ) return rc;
+  pCsr->depth = 1;
+  while(1){
+    sqlite3_bind_int(pStmt, 1, pgno);
+    rc = sqlite3_step(pStmt);
+    if( rc!=SQLITE_ROW ){
+      rc = SQLITE_ERROR;
+      break;
+    }
+    pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0);
+    aData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
+    if( aData==0 ){    
+      rc = SQLITE_NOMEM;
+      break;
+    }
+    if( pgno==1 ){
+      aData += 100;
+      pgsz -= 100;
+    }
+    pCsr->hasRowid = aData[0]!=2 && aData[0]!=10;
+    nCell = get_uint16(aData+3);
+    nEntry *= (nCell+1);
+    if( aData[0]==10 || aData[0]==13 ) break;
+    nPage *= (nCell+1);
+    if( nCell<=1 ){
+      pgno = get_uint32(aData+8);
+    }else{
+      iCell = get_uint16(aData+12+2*(nCell/2));
+      if( pgno==1 ) iCell -= 100;
+      if( iCell<=12 || iCell>=pgsz-4 ){
+        rc = SQLITE_CORRUPT;
+        break;
+      }
+      pgno = get_uint32(aData+iCell);
+    }
+    pCsr->depth++;
+    sqlite3_reset(pStmt);
+  }
+  sqlite3_finalize(pStmt);
+  pCsr->nPage = nPage;
+  pCsr->nEntry = nEntry;
+  if( rc==SQLITE_ROW ) rc = SQLITE_OK;
+  return rc;
+}
+
+/* Return a column for the sqlite_btreeinfo table */
+static int binfoColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){
+    int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1);
+    sqlite3 *db = sqlite3_context_db_handle(ctx);
+    int rc = binfoCompute(db, pgno, pCsr);
+    if( rc ){
+      return rc;
+    }
+  }
+  switch( i ){
+    case BINFO_COLUMN_NAME:
+    case BINFO_COLUMN_TYPE:
+    case BINFO_COLUMN_TBL_NAME:
+    case BINFO_COLUMN_ROOTPAGE:
+    case BINFO_COLUMN_SQL: {
+      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1));
+      break;
+    }
+    case BINFO_COLUMN_HASROWID: {
+      sqlite3_result_int(ctx, pCsr->hasRowid);
+      break;
+    }
+    case BINFO_COLUMN_NENTRY: {
+      sqlite3_result_int64(ctx, pCsr->nEntry);
+      break;
+    }
+    case BINFO_COLUMN_NPAGE: {
+      sqlite3_result_int(ctx, pCsr->nPage);
+      break;
+    }
+    case BINFO_COLUMN_DEPTH: {
+      sqlite3_result_int(ctx, pCsr->depth);
+      break;
+    }
+    case BINFO_COLUMN_SCHEMA: {
+      sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+  *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Invoke this routine to register the "sqlite_btreeinfo" virtual table module
+*/
+int sqlite3BinfoRegister(sqlite3 *db){
+  static sqlite3_module binfo_module = {
+    0,                           /* iVersion */
+    0,                           /* xCreate */
+    binfoConnect,                /* xConnect */
+    binfoBestIndex,              /* xBestIndex */
+    binfoDisconnect,             /* xDisconnect */
+    0,                           /* xDestroy */
+    binfoOpen,                   /* xOpen - open a cursor */
+    binfoClose,                  /* xClose - close a cursor */
+    binfoFilter,                 /* xFilter - configure scan constraints */
+    binfoNext,                   /* xNext - advance a cursor */
+    binfoEof,                    /* xEof - check for end of scan */
+    binfoColumn,                 /* xColumn - read data */
+    binfoRowid,                  /* xRowid - read data */
+    0,                           /* xUpdate */
+    0,                           /* xBegin */
+    0,                           /* xSync */
+    0,                           /* xCommit */
+    0,                           /* xRollback */
+    0,                           /* xFindMethod */
+    0,                           /* xRename */
+    0,                           /* xSavepoint */
+    0,                           /* xRelease */
+    0,                           /* xRollbackTo */
+  };
+  return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_btreeinfo_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return sqlite3BinfoRegister(db);
+}
diff --git a/manifest b/manifest
index d4ab8e50ee..c05d3b0728 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Force\sthe\sautoconf\smakefile\sfor\sMSVC\sto\salways\suse\sthe\s'find.exe'\sincluded\swith\sWindows.
-D 2017-10-24T21:17:12.950
+C Add\sthe\ssqlite_btreeinfo\seponymous-only\stable\sfor\sintrospecting\sthe\sschema\nand\sestimating\sthe\ssizes\sof\svarious\sbtrees.
+D 2017-10-25T01:34:24.293
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -258,6 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
+F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 w ext/misc/btreelist.c
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
@@ -1664,7 +1665,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5ab662b443df17c5ebdadc0bdac2f447c5c7e86f7a32a6943bb4ac8605879441
-R 54e49932e96b845a01a6eb1a3cc08724
-U mistachkin
-Z 580a702e0e85c423f77f3c922262f566
+P b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887
+R 2cb01c8a7e418d5768c813428fd16a20
+U drh
+Z 2d2691251ae11d16de493df9f4384426
diff --git a/manifest.uuid b/manifest.uuid
index 5f2c913791..fb6118cf93 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887
\ No newline at end of file
+1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d
\ No newline at end of file

From b5d013edd1f7fe1623b5951bb4664f6b23638f2f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 25 Oct 2017 16:14:12 +0000
Subject: [PATCH 213/270] Do not reference the ioctl() system call in the unix
 backend unless it is actually needed by the Batch Atomic Write extension. 
 This should allow the build to work on VxWorks.

FossilOrigin-Name: adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/os_unix.c |  4 ++++
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index c05d3b0728..29877dd0e2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite_btreeinfo\seponymous-only\stable\sfor\sintrospecting\sthe\sschema\nand\sestimating\sthe\ssizes\sof\svarious\sbtrees.
-D 2017-10-25T01:34:24.293
+C Do\snot\sreference\sthe\sioctl()\ssystem\scall\sin\sthe\sunix\sbackend\sunless\sit\nis\sactually\sneeded\sby\sthe\sBatch\sAtomic\sWrite\sextension.\s\sThis\sshould\sallow\nthe\sbuild\sto\swork\son\sVxWorks.
+D 2017-10-25T16:14:12.910
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -258,7 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 w ext/misc/btreelist.c
+F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
@@ -446,7 +446,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 3984fc069df59e26f000e30609611cecdb4e93293e6ee52313a473a7e874af1b
+F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1665,7 +1665,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b76bffd332585e8412a0a994ae6dee79a83213d8b709d7f858c5c05678ab0887
-R 2cb01c8a7e418d5768c813428fd16a20
+P 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d
+R 15e40e111eafb5e48910635a49cabea3
 U drh
-Z 2d2691251ae11d16de493df9f4384426
+Z 02638cae8d6ec29df692272f34f9ba5b
diff --git a/manifest.uuid b/manifest.uuid
index fb6118cf93..b582a39fb7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d
\ No newline at end of file
+adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 4445104dd6..c33d21fd9b 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -513,7 +513,11 @@ static struct unix_syscall {
 #endif
 #define osLstat      ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent)
 
+#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE)
   { "ioctl",         (sqlite3_syscall_ptr)ioctl,          0 },
+#else
+  { "ioctl",         (sqlite3_syscall_ptr)0,              0 },
+#endif
 #define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent)
 
 }; /* End of the overrideable system calls */

From 1917e92fdb0084faceb44dc06694aa13b051843b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 25 Oct 2017 16:38:34 +0000
Subject: [PATCH 214/270] Add SQL scalar function rtreecheck() to the rtree
 module. For running checks to ensure the shadow tables used by an rtree
 virtual table are internally consistent.

FossilOrigin-Name: dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327
---
 ext/rtree/rtree.c         | 424 +++++++++++++++++++++++++++++++++++++-
 ext/rtree/rtree1.test     |   2 +-
 ext/rtree/rtree2.test     |   8 +-
 ext/rtree/rtree4.test     |   2 +
 ext/rtree/rtree5.test     |   2 +
 ext/rtree/rtree7.test     |   3 +
 ext/rtree/rtree8.test     |   6 +-
 ext/rtree/rtree9.test     |   6 +-
 ext/rtree/rtreeA.test     |  26 ++-
 ext/rtree/rtreeB.test     |   3 +
 ext/rtree/rtreeC.test     |   2 +
 ext/rtree/rtreeE.test     |   5 +-
 ext/rtree/rtreeF.test     |   3 +
 ext/rtree/rtreeG.test     |   2 +
 ext/rtree/rtree_util.tcl  |   5 +
 ext/rtree/rtreecheck.test | 113 ++++++++++
 manifest                  |  43 ++--
 manifest.uuid             |   2 +-
 18 files changed, 623 insertions(+), 34 deletions(-)
 create mode 100644 ext/rtree/rtreecheck.test

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index 32b9b3027a..500d922102 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -209,7 +209,7 @@ struct RtreeSearchPoint {
 ** The smallest possible node-size is (512-64)==448 bytes. And the largest
 ** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates).
 ** Therefore all non-root nodes must contain at least 3 entries. Since 
-** 2^40 is greater than 2^64, an r-tree structure always has a depth of
+** 3^40 is greater than 2^64, an r-tree structure always has a depth of
 ** 40 or less.
 */
 #define RTREE_MAX_DEPTH 40
@@ -3608,6 +3608,425 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
   }
 }
 
+/*
+** Context object passed between the various routines that make up the
+** implementation of integrity-check function rtreecheck().
+*/
+typedef struct RtreeCheck RtreeCheck;
+struct RtreeCheck {
+  sqlite3 *db;                    /* Database handle */
+  const char *zDb;                /* Database containing rtree table */
+  const char *zTab;               /* Name of rtree table */
+  int bInt;                       /* True for rtree_i32 table */
+  int nDim;                       /* Number of dimensions for this rtree tbl */
+  sqlite3_stmt *pGetNode;         /* Statement used to retrieve nodes */
+  sqlite3_stmt *aCheckMapping[2]; /* Statements to query %_parent/%_rowid */
+  int nLeaf;                      /* Number of leaf cells in table */
+  int nNonLeaf;                   /* Number of non-leaf cells in table */
+  int rc;                         /* Return code */
+  char *zReport;                  /* Message to report */
+  int nErr;                       /* Number of lines in zReport */
+};
+
+#define RTREE_CHECK_MAX_ERROR 100
+
+/*
+** Reset SQL statement pStmt. If the sqlite3_reset() call returns an error,
+** and RtreeCheck.rc==SQLITE_OK, set RtreeCheck.rc to the error code.
+*/
+static void rtreeCheckReset(RtreeCheck *pCheck, sqlite3_stmt *pStmt){
+  int rc = sqlite3_reset(pStmt);
+  if( pCheck->rc==SQLITE_OK ) pCheck->rc = rc;
+}
+
+/*
+** The second and subsequent arguments to this function are a format string
+** and printf style arguments. This function formats the string and attempts
+** to compile it as an SQL statement.
+**
+** If successful, a pointer to the new SQL statement is returned. Otherwise,
+** NULL is returned and an error code left in RtreeCheck.rc.
+*/
+static sqlite3_stmt *rtreeCheckPrepare(
+  RtreeCheck *pCheck,             /* RtreeCheck object */
+  const char *zFmt, ...           /* Format string and trailing args */
+){
+  va_list ap;
+  va_start(ap, zFmt);
+  char *z = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_stmt *pRet = 0;
+
+  if( pCheck->rc==SQLITE_OK ){
+    pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0);
+  }
+
+  sqlite3_free(z);
+  va_end(ap);
+  return pRet;
+}
+
+/*
+** The second and subsequent arguments to this function are a printf()
+** style format string and arguments. This function formats the string and
+** appends it to the report being accumuated in pCheck.
+*/
+static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  if( pCheck->rc==SQLITE_OK && pCheck->nErr<RTREE_CHECK_MAX_ERROR ){
+    char *z = sqlite3_vmprintf(zFmt, ap);
+    if( z==0 ){
+      pCheck->rc = SQLITE_NOMEM;
+    }else{
+      pCheck->zReport = sqlite3_mprintf("%z%s%z", 
+          pCheck->zReport, (pCheck->zReport ? "\n" : ""), z
+      );
+      if( pCheck->zReport==0 ){
+        pCheck->rc = SQLITE_NOMEM;
+      }
+    }
+    pCheck->nErr++;
+  }
+  va_end(ap);
+}
+
+/*
+** This function is a no-op if there is already an error code stored
+** in the RtreeCheck object indicated by the first argument. NULL is
+** returned in this case.
+**
+** Otherwise, the contents of rtree table node iNode are loaded from
+** the database and copied into a buffer obtained from sqlite3_malloc().
+** If no error occurs, a pointer to the buffer is returned and (*pnNode)
+** is set to the size of the buffer in bytes.
+**
+** Or, if an error does occur, NULL is returned and an error code left
+** in the RtreeCheck object. The final value of *pnNode is undefined in
+** this case.
+*/
+static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){
+  u8 *pRet = 0;                   /* Return value */
+
+  assert( pCheck->rc==SQLITE_OK );
+  if( pCheck->pGetNode==0 ){
+    pCheck->pGetNode = rtreeCheckPrepare(pCheck,
+        "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", 
+        pCheck->zDb, pCheck->zTab
+    );
+  }
+
+  if( pCheck->rc==SQLITE_OK ){
+    sqlite3_bind_int64(pCheck->pGetNode, 1, iNode);
+    if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){
+      int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0);
+      const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0);
+      pRet = sqlite3_malloc(nNode);
+      if( pRet==0 ){
+        pCheck->rc = SQLITE_NOMEM;
+      }else{
+        memcpy(pRet, pNode, nNode);
+        *pnNode = nNode;
+      }
+    }
+    rtreeCheckReset(pCheck, pCheck->pGetNode);
+    if( pCheck->rc==SQLITE_OK && pRet==0 ){
+      rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode);
+    }
+  }
+
+  return pRet;
+}
+
+/*
+** This function is used to check that the %_parent (if bLeaf==0) or %_rowid
+** (if bLeaf==1) table contains a specified entry. The schemas of the
+** two tables are:
+**
+**   CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER)
+**   CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER)
+**
+** In both cases, this function checks that there exists an entry with
+** IPK value iKey and the second column set to iVal.
+**
+*/
+static void rtreeCheckMapping(
+  RtreeCheck *pCheck,             /* RtreeCheck object */
+  int bLeaf,                      /* True for a leaf cell, false for interior */
+  i64 iKey,                       /* Key for mapping */
+  i64 iVal                        /* Expected value for mapping */
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char *azSql[2] = {
+    "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?",
+    "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?"
+  };
+
+  assert( bLeaf==0 || bLeaf==1 );
+  if( pCheck->aCheckMapping[bLeaf]==0 ){
+    pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck,
+        azSql[bLeaf], pCheck->zDb, pCheck->zTab
+    );
+  }
+  if( pCheck->rc!=SQLITE_OK ) return;
+
+  pStmt = pCheck->aCheckMapping[bLeaf];
+  sqlite3_bind_int64(pStmt, 1, iKey);
+  rc = sqlite3_step(pStmt);
+  if( rc==SQLITE_DONE ){
+    rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table",
+        iKey, iVal, (bLeaf ? "%_rowid" : "%_parent")
+    );
+  }else if( rc==SQLITE_ROW ){
+    i64 ii = sqlite3_column_int64(pStmt, 0);
+   if( ii!=iVal ){
+      rtreeCheckAppendMsg(pCheck, 
+          "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)",
+          iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal
+      );
+    }
+  }
+  rtreeCheckReset(pCheck, pStmt);
+}
+
+static void rtreeCheckCellCoord(
+  RtreeCheck *pCheck, 
+  i64 iNode,
+  int iCell,
+  u8 *pCell,                      /* Pointer to cell coordinates */
+  u8 *pParent                     /* Pointer to parent coordinates */
+){
+  RtreeCoord c1, c2;
+  RtreeCoord p1, p2;
+  int i;
+
+  for(i=0; i<pCheck->nDim; i++){
+    readCoord(&pCell[4*2*i], &c1);
+    readCoord(&pCell[4*(2*i + 1)], &c2);
+
+    /* printf("%e, %e\n", c1.u.f, c2.u.f); */
+    if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){
+      rtreeCheckAppendMsg(pCheck, 
+          "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode
+      );
+    }
+
+    if( pParent ){
+      readCoord(&pParent[4*2*i], &p1);
+      readCoord(&pParent[4*(2*i + 1)], &p2);
+
+      if( (pCheck->bInt ? c1.i<p1.i : c1.f<p1.f) 
+       || (pCheck->bInt ? c2.i>p2.i : c2.f>p2.f)
+      ){
+        rtreeCheckAppendMsg(pCheck, 
+            "Dimension %d of cell %d on node %lld is corrupt relative to parent"
+            , i, iCell, iNode
+        );
+      }
+    }
+  }
+}
+
+static void rtreeCheckNode(
+  RtreeCheck *pCheck,
+  int iDepth,                     /* Depth of iNode (0==leaf) */
+  u8 *aParent,                    /* Buffer containing parent coords */
+  i64 iNode                       /* Node to check */
+){
+  u8 *aNode = 0;
+  int nNode = 0;
+
+  assert( iNode==1 || aParent!=0 );
+  assert( pCheck->nDim>0 );
+
+  aNode = rtreeCheckGetNode(pCheck, iNode, &nNode);
+  if( aNode ){
+    if( nNode<4 ){
+      rtreeCheckAppendMsg(pCheck, 
+          "Node %lld is too small (%d bytes)", iNode, nNode
+      );
+    }else{
+      int nCell;                  /* Number of cells on page */
+      int i;                      /* Used to iterate through cells */
+      if( aParent==0 ){
+        iDepth = readInt16(aNode);
+        if( iDepth>RTREE_MAX_DEPTH ){
+          rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth);
+          sqlite3_free(aNode);
+          return;
+        }
+      }
+      nCell = readInt16(&aNode[2]);
+      if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){
+        rtreeCheckAppendMsg(pCheck, 
+            "Node %lld is too small for cell count of %d (%d bytes)", 
+            iNode, nCell, nNode
+        );
+      }
+      for(i=0; i<nCell; i++){
+        u8 *pCell = &aNode[4 + i*(8 + pCheck->nDim*2*4)];
+        i64 iVal = readInt64(pCell);
+        rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent);
+
+        if( iDepth>0 ){
+          rtreeCheckMapping(pCheck, 0, iVal, iNode);
+          rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal);
+          pCheck->nNonLeaf++;
+        }else{
+          rtreeCheckMapping(pCheck, 1, iVal, iNode);
+          pCheck->nLeaf++;
+        }
+      }
+    }
+    sqlite3_free(aNode);
+  }
+}
+
+static void rtreeCheckCount(
+  RtreeCheck *pCheck, const char *zTbl, i64 nExpected
+){
+  if( pCheck->rc==SQLITE_OK ){
+    sqlite3_stmt *pCount;
+    pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'",
+        pCheck->zDb, pCheck->zTab, zTbl
+    );
+    if( pCount ){
+      if( sqlite3_step(pCount)==SQLITE_ROW ){
+        i64 nActual = sqlite3_column_int64(pCount, 0);
+        if( nActual!=nExpected ){
+          rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table"
+              " - expected %lld, actual %lld" , zTbl, nExpected, nActual
+          );
+        }
+      }
+      pCheck->rc = sqlite3_finalize(pCount);
+    }
+  }
+}
+
+static int rtreeCheck(
+  sqlite3 *db,                    /* Database handle to access db through */
+  const char *zDb,                /* Name of db ("main", "temp" etc.) */
+  const char *zTab,               /* Name of rtree table to check */
+  char **pzReport                 /* OUT: sqlite3_malloc'd report text */
+){
+  RtreeCheck check;               /* Common context for various routines */
+  sqlite3_stmt *pStmt = 0;        /* Used to find column count of rtree table */
+  int bEnd = 0;                   /* True if transaction should be closed */
+
+  /* Initialize the context object */
+  memset(&check, 0, sizeof(check));
+  check.db = db;
+  check.zDb = zDb;
+  check.zTab = zTab;
+
+  /* If there is not already an open transaction, open one now. This is
+  ** to ensure that the queries run as part of this integrity-check operate
+  ** on a consistent snapshot.  */
+  if( sqlite3_get_autocommit(db) ){
+    check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+    bEnd = 1;
+  }
+
+  /* Find number of dimensions in the rtree table. */
+  pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.%Q", zDb, zTab);
+  if( pStmt ){
+    int rc;
+    check.nDim = (sqlite3_column_count(pStmt) - 1) / 2;
+    if( check.nDim<1 ){
+      rtreeCheckAppendMsg(&check, "Schema corrupt or not an rtree");
+    }else if( SQLITE_ROW==sqlite3_step(pStmt) ){
+      check.bInt = (sqlite3_column_type(pStmt, 1)==SQLITE_INTEGER);
+    }
+    rc = sqlite3_finalize(pStmt);
+    if( rc!=SQLITE_CORRUPT ) check.rc = rc;
+  }
+
+  /* Do the actual integrity-check */
+  if( check.rc==SQLITE_OK ){
+    rtreeCheckNode(&check, 0, 0, 1);
+  }
+  rtreeCheckCount(&check, "_rowid", check.nLeaf);
+  rtreeCheckCount(&check, "_parent", check.nNonLeaf);
+
+  /* Finalize SQL statements used by the integrity-check */
+  sqlite3_finalize(check.pGetNode);
+  sqlite3_finalize(check.aCheckMapping[0]);
+  sqlite3_finalize(check.aCheckMapping[1]);
+
+  /* If one was opened, close the transaction */
+  if( bEnd ){
+    int rc = sqlite3_exec(db, "END", 0, 0, 0);
+    if( check.rc==SQLITE_OK ) check.rc = rc;
+  }
+  *pzReport = check.zReport;
+  return check.rc;
+}
+
+/*
+** Usage:
+**
+**   rtreecheck(<rtree-table>);
+**   rtreecheck(<database>, <rtree-table>);
+**
+** Invoking this SQL function runs an integrity-check on the named rtree
+** table. The integrity-check verifies the following:
+**
+**   1. For each cell in the r-tree structure (%_node table), that:
+**
+**       a) for each dimension, (coord1 <= coord2).
+**
+**       b) unless the cell is on the root node, that the cell is bounded
+**          by the parent cell on the parent node.
+**
+**       c) for leaf nodes, that there is an entry in the %_rowid 
+**          table corresponding to the cell's rowid value that 
+**          points to the correct node.
+**
+**       d) for cells on non-leaf nodes, that there is an entry in the 
+**          %_parent table mapping from the cell's child node to the
+**          node that it resides on.
+**
+**   2. That there are the same number of entries in the %_rowid table
+**      as there are leaf cells in the r-tree structure, and that there
+**      is a leaf cell that corresponds to each entry in the %_rowid table.
+**
+**   3. That there are the same number of entries in the %_parent table
+**      as there are non-leaf cells in the r-tree structure, and that 
+**      there is a non-leaf cell that corresponds to each entry in the 
+**      %_parent table.
+*/
+static void rtreecheck(
+  sqlite3_context *ctx, 
+  int nArg, 
+  sqlite3_value **apArg
+){
+  if( nArg!=1 && nArg!=2 ){
+    sqlite3_result_error(ctx, 
+        "wrong number of arguments to function rtreecheck()", -1
+    );
+  }else{
+    int rc;
+    char *zReport = 0;
+    const char *zDb = (const char*)sqlite3_value_text(apArg[0]);
+    const char *zTab;
+    if( nArg==1 ){
+      zTab = zDb;
+      zDb = "main";
+    }else{
+      zTab = (const char*)sqlite3_value_text(apArg[1]);
+    }
+    rc = rtreeCheck(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport);
+    if( rc==SQLITE_OK ){
+      sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT);
+    }else{
+      sqlite3_result_error_code(ctx, rc);
+    }
+    sqlite3_free(zReport);
+  }
+}
+
+
 /*
 ** Register the r-tree module with database handle db. This creates the
 ** virtual table module "rtree" and the debugging/analysis scalar 
@@ -3621,6 +4040,9 @@ int sqlite3RtreeInit(sqlite3 *db){
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "rtreecheck", -1, utf8, 0,rtreecheck, 0,0);
+  }
   if( rc==SQLITE_OK ){
 #ifdef SQLITE_RTREE_INT_ONLY
     void *c = (void *)RTREE_COORD_INT32;
diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test
index becc8e1a97..0deee6635b 100644
--- a/ext/rtree/rtree1.test
+++ b/ext/rtree/rtree1.test
@@ -519,7 +519,7 @@ foreach {tn sql_template testdata} {
     do_test $testname.2 [list sql_uses_stmt db $sql] $uses
     do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data
 
-    do_test $testname.4 { rtree_check db t1 } 0
+    do_rtree_integrity_test $testname.4 t1
     db close
   }
 }
diff --git a/ext/rtree/rtree2.test b/ext/rtree/rtree2.test
index f5d15cc6b9..853737b9c7 100644
--- a/ext/rtree/rtree2.test
+++ b/ext/rtree/rtree2.test
@@ -81,9 +81,7 @@ foreach module {rtree_i32 rtree} {
       set rc
     } {1}
   
-    do_test rtree2-$module.$nDim.3 {
-      rtree_check db t1
-    } 0
+    do_rtree_integrity_test rtree2-$module.$nDim.3 t1
   
     set OPS [list < > <= >= =]
     for {set ii 0} {$ii < $::NSELECT} {incr ii} {
@@ -133,9 +131,7 @@ foreach module {rtree_i32 rtree} {
         }
         set rc
       } {1}
-      do_test rtree2-$module.$nDim.5.$ii.2 {
-        rtree_check db t1
-      } {0}
+      do_rtree_integrity_test rtree2-$module.$nDim.5.$ii.2 t1
     }
   
     do_test rtree2-$module.$nDim.6 {
diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test
index a3872b0735..af3f8d3995 100644
--- a/ext/rtree/rtree4.test
+++ b/ext/rtree/rtree4.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 
 ifcapable !rtree {
@@ -246,6 +247,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
   }
 
+  do_rtree_integrity_test rtree4-$nDim.3 rx
 }
 
 finish_test
diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test
index 8ff90b0cb7..749385e882 100644
--- a/ext/rtree/rtree5.test
+++ b/ext/rtree/rtree5.test
@@ -16,6 +16,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 
 ifcapable !rtree {
@@ -76,5 +77,6 @@ do_test rtree5-1.13 {
         y1=-2147483648 AND y2=-2147483643
   }
 } {2 2147483643 2147483647 -2147483648 -2147483643}
+do_rtree_integrity_test rtree5-1.14 t1
 
 finish_test
diff --git a/ext/rtree/rtree7.test b/ext/rtree/rtree7.test
index 4eee4c219a..1556179c4f 100644
--- a/ext/rtree/rtree7.test
+++ b/ext/rtree/rtree7.test
@@ -17,6 +17,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 
 ifcapable !rtree||!vacuum {
@@ -67,4 +68,6 @@ do_test rtree7-1.5 {
   }
 } {51 102 153 204}
 
+do_rtree_integrity_test rtree7-1.6 rt
+
 finish_test
diff --git a/ext/rtree/rtree8.test b/ext/rtree/rtree8.test
index 024d098e07..ac18fb477d 100644
--- a/ext/rtree/rtree8.test
+++ b/ext/rtree/rtree8.test
@@ -14,6 +14,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 
@@ -64,6 +65,7 @@ do_test rtree8-1.2.2 { nested_select 1 } {51}
 # nodes internally.
 #
 populate_t1 1500
+do_rtree_integrity_test rtree8-1.3.0 t1
 do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {164}
 do_test rtree8-1.3.2 {
   set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}]
@@ -158,13 +160,15 @@ do_test rtree8-5.2 {
   }
   execsql COMMIT
 } {}
-do_test rtree8-5.3 {
+do_rtree_integrity_test rtree8-5.3 t2
+do_test rtree8-5.4 {
   execsql BEGIN
   for {set i 0} {$i < 200} {incr i} {
     execsql { DELETE FROM t2 WHERE id = $i }
   }
   execsql COMMIT
 } {}
+do_rtree_integrity_test rtree8-5.5 t2
 
 
 finish_test
diff --git a/ext/rtree/rtree9.test b/ext/rtree/rtree9.test
index b2361b2bd0..a7cd344d76 100644
--- a/ext/rtree/rtree9.test
+++ b/ext/rtree/rtree9.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 ifcapable rtree_int_only { finish_test; return }
@@ -42,6 +43,7 @@ for {set i 0} {$i < 1000} {incr i} {
   set z [expr ($i/100)%10]
   execsql { INSERT INTO rt VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
 }
+do_rtree_integrity_test rtree9-2.0 rt
 do_execsql_test rtree9-2.1 {
   SELECT id FROM rt WHERE id MATCH cube(2.5, 2.5, 2.5, 1, 1, 1) ORDER BY id;
 } {222 223 232 233 322 323 332 333}
@@ -50,7 +52,7 @@ do_execsql_test rtree9-2.2 {
 } {555 556 565 566 655 656 665 666}
 
 
-do_execsql_test rtree9-3.1 {
+do_execsql_test rtree9-3.0 {
   CREATE VIRTUAL TABLE rt32 USING rtree_i32(id, x1, x2, y1, y2, z1, z2);
 } {} 
 for {set i 0} {$i < 1000} {incr i} {
@@ -59,6 +61,7 @@ for {set i 0} {$i < 1000} {incr i} {
   set z [expr ($i/100)%10]
   execsql { INSERT INTO rt32 VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
 }
+do_rtree_integrity_test rtree9-3.1 rt32
 do_execsql_test rtree9-3.2 {
   SELECT id FROM rt32 WHERE id MATCH cube(3, 3, 3, 1, 1, 1) ORDER BY id;
 } {222 223 224 232 233 234 242 243 244 322 323 324 332 333 334 342 343 344 422 423 424 432 433 434 442 443 444}
@@ -121,5 +124,6 @@ do_execsql_test rtree9-5.3 {
   UPDATE rt2 SET xmin=xmin+5, ymin=ymin+5, xmax=xmax+5, ymax=ymax+5;
   SELECT id FROM rt2 WHERE id MATCH circle(5.0, 5.0, 2.0);
 } {1 2 3 4 13 14 15 16 17}
+do_rtree_integrity_test rtree9-5.4 rt2
 
 finish_test
diff --git a/ext/rtree/rtreeA.test b/ext/rtree/rtreeA.test
index d583a15264..190d7d7532 100644
--- a/ext/rtree/rtreeA.test
+++ b/ext/rtree/rtreeA.test
@@ -108,6 +108,12 @@ do_corruption_tests rtreeA-1.1 {
   4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
 }
 
+do_execsql_test rtreeA-1.1.1 {
+  SELECT rtreecheck('main', 't1')
+} {{Node 1 missing from database
+Wrong number of entries in %_rowid table - expected 0, actual 500
+Wrong number of entries in %_parent table - expected 0, actual 23}}
+
 do_execsql_test  rtreeA-1.2.0 { DROP TABLE t1_node } {}
 do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" {
   1   "SELECT * FROM t1"
@@ -157,6 +163,10 @@ do_corruption_tests rtreeA-3.1 {
   3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
 }
 
+do_execsql_test rtreeA-3.1.0.3 {
+  SELECT rtreecheck('main', 't1')!="ok"
+} {1}
+
 do_test rtreeA-3.2.0 { set_tree_depth t1 1000 } {1000}
 do_corruption_tests rtreeA-3.2 {
   1   "SELECT * FROM t1"
@@ -176,6 +186,12 @@ do_corruption_tests rtreeA-3.3 {
   3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
 }
 
+do_execsql_test rtreeA-3.3.3.4 {
+  SELECT rtreecheck('main', 't1')
+} {{Rtree depth out of range (65535)
+Wrong number of entries in %_rowid table - expected 0, actual 499
+Wrong number of entries in %_parent table - expected 0, actual 23}}
+
 #-------------------------------------------------------------------------
 # Set the "number of entries" field on some nodes incorrectly.
 #
@@ -203,6 +219,10 @@ do_corruption_tests rtreeA-5.1 {
   2   "DELETE FROM t1"
 }
 
+do_execsql_test rtreeA-5.2 {
+  SELECT rtreecheck('main', 't1')!="ok"
+} {1}
+
 #-------------------------------------------------------------------------
 # Add some bad entries to the %_parent table.
 #
@@ -216,6 +236,10 @@ do_corruption_tests rtreeA-6.1 {
   2   "UPDATE t1 SET x1=x1+1, x2=x2+1"
 }
 
+do_execsql_test rtreeA-6.2 {
+  SELECT rtreecheck('main', 't1')!="ok"
+} {1}
+
 #-------------------------------------------------------------------------
 # Truncated blobs in the _node table.
 #
@@ -233,5 +257,5 @@ do_test rtreeA-7.120 {
 } {SQLITE_CORRUPT_VTAB}
 
 
-
 finish_test
+
diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test
index aeb308eca7..6fc31042ca 100644
--- a/ext/rtree/rtreeB.test
+++ b/ext/rtree/rtreeB.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 
@@ -44,4 +45,6 @@ ifcapable rtree_int_only {
   } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}}
 }
 
+do_rtree_integrity_test rtreeB-1.2 t1
+
 finish_test
diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test
index a26c401e0d..a5e5b97f96 100644
--- a/ext/rtree/rtreeC.test
+++ b/ext/rtree/rtreeC.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 set testprefix rtreeC
@@ -180,6 +181,7 @@ do_execsql_test 5.1 {
 
   INSERT INTO rt SELECT x, x, x+1 FROM t1 WHERE x<=5;
 }
+do_rtree_integrity_test 5.1.1 rt
 
 # First test a query with no ANALYZE data at all. The outer loop is
 # real table "t1".
diff --git a/ext/rtree/rtreeE.test b/ext/rtree/rtreeE.test
index 3e5ba3a67f..72dcc94c9f 100644
--- a/ext/rtree/rtreeE.test
+++ b/ext/rtree/rtreeE.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 ifcapable rtree_int_only { finish_test; return }
@@ -25,7 +26,7 @@ ifcapable rtree_int_only { finish_test; return }
 #
 register_circle_geom db
 
-do_execsql_test rtreeE-1.1 {
+do_execsql_test rtreeE-1.0.0 {
   PRAGMA page_size=512;
   CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1);
   
@@ -47,6 +48,7 @@ do_execsql_test rtreeE-1.1 {
     y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4)
   INSERT INTO rt1 SELECT 200+x+5*y, x*7, x*7+15, y*7+200, y*7+215 FROM x, y;
 } {}
+do_rtree_integrity_test rtreeE-1.0.1 rt1
 
 # Queries against each of the three clusters */
 do_execsql_test rtreeE-1.1 {
@@ -111,6 +113,7 @@ do_test rtreeE-2.1 {
     COMMIT;
   }
 } {}
+do_rtree_integrity_test rtreeE-2.1.1 rt2
 
 for {set i 1} {$i<=200} {incr i} {
   set dx [expr {int(rand()*100)}]
diff --git a/ext/rtree/rtreeF.test b/ext/rtree/rtreeF.test
index c9620d34f7..561770d085 100644
--- a/ext/rtree/rtreeF.test
+++ b/ext/rtree/rtreeF.test
@@ -28,6 +28,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 
@@ -78,4 +79,6 @@ do_execsql_test rtreeF-1.5 {
   SELECT y FROM t2 ORDER BY y;
 } {1 4 5 | 1 4}
 
+do_rtree_integrity_test rtreeF-1.6 t3
+
 finish_test
diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test
index bffd17ecaa..3bef89c8e7 100644
--- a/ext/rtree/rtreeG.test
+++ b/ext/rtree/rtreeG.test
@@ -15,6 +15,7 @@
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. test]
 } 
+source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
 ifcapable !rtree { finish_test ; return }
 
@@ -37,6 +38,7 @@ do_execsql_test rtreeG-1.2 {
   INSERT INTO t1 VALUES(1,10,15,5,23),(2,20,21,5,23),(3,10,15,20,30);
   SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25;
 } {1}
+do_rtree_integrity_test rtreeG-1.2.integrity t1
 do_test rtreeG-1.2log {
   set ::log
 } {}
diff --git a/ext/rtree/rtree_util.tcl b/ext/rtree/rtree_util.tcl
index 50a1b58065..afa588e4e9 100644
--- a/ext/rtree/rtree_util.tcl
+++ b/ext/rtree/rtree_util.tcl
@@ -190,3 +190,8 @@ proc rtree_treedump {db zTab} {
   set d [rtree_depth $db $zTab]
   rtree_nodetreedump $db $zTab "" $d 1
 }
+
+proc do_rtree_integrity_test {tn tbl} {
+  uplevel [list do_execsql_test $tn "SELECT rtreecheck('$tbl')" ok]
+}
+
diff --git a/ext/rtree/rtreecheck.test b/ext/rtree/rtreecheck.test
new file mode 100644
index 0000000000..eb0e8f0acb
--- /dev/null
+++ b/ext/rtree/rtreecheck.test
@@ -0,0 +1,113 @@
+# 2017 August 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+set testprefix rtreecheck
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+proc swap_int32 {blob i0 i1} {
+  binary scan $blob I* L
+
+  set a [lindex $L $i0]
+  set b [lindex $L $i1]
+
+  lset L $i0 $b
+  lset L $i1 $a
+
+  binary format I* $L
+}
+
+do_catchsql_test 1.0 {
+  SELECT rtreecheck();
+} {1 {wrong number of arguments to function rtreecheck()}}
+
+do_catchsql_test 1.1 {
+  SELECT rtreecheck(0,0,0);
+} {1 {wrong number of arguments to function rtreecheck()}}
+
+
+proc setup_simple_db {{module rtree}} {
+  reset_db
+  db func swap_int32 swap_int32
+  execsql "
+    CREATE VIRTUAL TABLE r1 USING $module (id, x1, x2, y1, y2);
+    INSERT INTO r1 VALUES(1,  5, 5, 5, 5);  --  3
+    INSERT INTO r1 VALUES(2,  6, 6, 6, 6);  --  9
+    INSERT INTO r1 VALUES(3,  7, 7, 7, 7);  -- 15
+    INSERT INTO r1 VALUES(4,  8, 8, 8, 8);  -- 21
+    INSERT INTO r1 VALUES(5,  9, 9, 9, 9);  -- 27
+  "
+}
+
+setup_simple_db
+do_execsql_test 2.1 { 
+  SELECT rtreecheck('r1') 
+} {ok}
+
+do_execsql_test 2.2 {
+  UPDATE r1_node SET data = swap_int32(data, 3, 9);
+  UPDATE r1_node SET data = swap_int32(data, 23, 29);
+}
+
+do_execsql_test 2.3 { 
+  SELECT rtreecheck('r1') 
+} {{Dimension 0 of cell 0 on node 1 is corrupt
+Dimension 1 of cell 3 on node 1 is corrupt}}
+
+setup_simple_db
+do_execsql_test 2.4 {
+  DELETE FROM r1_rowid WHERE rowid = 3;
+  SELECT rtreecheck('r1') 
+} {{Mapping (3 -> 1) missing from %_rowid table
+Wrong number of entries in %_rowid table - expected 5, actual 4}}
+
+setup_simple_db
+do_execsql_test 2.5 {
+  UPDATE r1_rowid SET nodeno=2 WHERE rowid=3;
+  SELECT rtreecheck('r1') 
+} {{Found (3 -> 2) in %_rowid table, expected (3 -> 1)}}
+
+################
+reset_db
+do_execsql_test 3.0 { 
+  CREATE VIRTUAL TABLE r1 USING rtree_i32(id, x1, x2);
+  INSERT INTO r1 VALUES(1, 0x7FFFFFFF*-1, 0x7FFFFFFF);
+  INSERT INTO r1 VALUES(2, 0x7FFFFFFF*-1, 5);
+  INSERT INTO r1 VALUES(3, -5, 5);
+  INSERT INTO r1 VALUES(4, 5, 0x11111111);
+  INSERT INTO r1 VALUES(5, 5, 0x00800000);
+  INSERT INTO r1 VALUES(6, 5, 0x00008000);
+  INSERT INTO r1 VALUES(7, 5, 0x00000080);
+  INSERT INTO r1 VALUES(8, 5, 0x40490fdb);
+  INSERT INTO r1 VALUES(9, 0x7f800000, 0x7f900000);
+  SELECT rtreecheck('r1') 
+} {ok}
+
+breakpoint
+do_execsql_test 3.1 { 
+  CREATE VIRTUAL TABLE r2 USING rtree_i32(id, x1, x2);
+  INSERT INTO r2 VALUES(2, -1*(1<<31), -1*(1<<31)+5);
+  SELECT rtreecheck('r2') 
+} {ok}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index 29877dd0e2..57d11641e5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sreference\sthe\sioctl()\ssystem\scall\sin\sthe\sunix\sbackend\sunless\sit\nis\sactually\sneeded\sby\sthe\sBatch\sAtomic\sWrite\sextension.\s\sThis\sshould\sallow\nthe\sbuild\sto\swork\son\sVxWorks.
-D 2017-10-25T16:14:12.910
+C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\nto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\nconsistent.
+D 2017-10-25T16:38:34.144
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -329,26 +329,27 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c f2fd34db37ea053798f8e66b44a473449b21301d2b92505ee576823789e909fb
+F ext/rtree/rtree.c 7941c4283bef9ecdd1b7a5c20122006b7d363836468fe7f4bef46698c55d1d93
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
-F ext/rtree/rtree1.test 4fdd60ae034e43f2fefc26492032d02e742e8b14d468b7c51d95a1e2fa47cf00
-F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
+F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
+F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482
-F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0
-F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e
+F ext/rtree/rtree4.test 67b021858ba4334c8d49b3449476942c2ce0e5ef7123538f2e9dd508ed03a12d
+F ext/rtree/rtree5.test 8aaa4bcdc42f718fe165572f5623e4732831aca95a2bc32482d33d4d2cf1325d
 F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196
-F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971
-F ext/rtree/rtree8.test 076d9d5b783b61b7a23a5ab45fc899551dfffd821974f36ee599ff29f4de7a61
-F ext/rtree/rtree9.test 8bfa84dfaba1c897468a2448c28db0a00ad12d464225b5993c7814e907f3776f
-F ext/rtree/rtreeA.test c09ad3f76c08feac00770685ff50ca12966dc0c641bf19a982b26a80643b46d1
-F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e
-F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d
+F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5
+F ext/rtree/rtree8.test 649f5a37ec656028a4a32674b9b1183104285a7625a09d2a8f52a1cef72c93f2
+F ext/rtree/rtree9.test c646f12c8c1c68ef015c6c043d86a0c42488e2e68ed1bb1b0771a7ca246cbabf
+F ext/rtree/rtreeA.test 20623ca337ca3bd7e008cc9fb49e44dbe97f1a80b238e10a12bb4afcd0da3776
+F ext/rtree/rtreeB.test 4cec297f8e5c588654bbf3c6ed0903f10612be8a2878055dd25faf8c71758bc9
+F ext/rtree/rtreeC.test d9d06dda1aee68b4dc227dfcc899f335f8b621e9d1920ee3d4e5dab8ccd71db7
 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc
-F ext/rtree/rtreeE.test 45a147a64a76306172819562309681d8e90f94bb
-F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4
-F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1
+F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9
+F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331
+F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
-F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea
+F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed
+F ext/rtree/rtreecheck.test f610cb77ca1ba611e656018a7d960cd46054baecd2f12d1149bf1fec121aa230
 F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d
 F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28
 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
@@ -1665,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1e30f4772db1e1086096f72d32e87c552923be8b264aa13cf822fae754eb083d
-R 15e40e111eafb5e48910635a49cabea3
-U drh
-Z 02638cae8d6ec29df692272f34f9ba5b
+P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673
+R b2ab5ba3cbab5da53d273c6ee124bea0
+U dan
+Z 3064e6e3a20de6115706024077565624
diff --git a/manifest.uuid b/manifest.uuid
index b582a39fb7..d2644e0a82 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673
\ No newline at end of file
+dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327
\ No newline at end of file

From 8c66e5b73e0424ada2b96e340058be96b80c2860 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 25 Oct 2017 18:00:58 +0000
Subject: [PATCH 215/270] Fix compilation issue (C99-ism) seen with MSVC.

FossilOrigin-Name: 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5
---
 ext/rtree/rtree.c |  6 ++++--
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index 500d922102..c7bf671c96 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -3652,10 +3652,12 @@ static sqlite3_stmt *rtreeCheckPrepare(
   const char *zFmt, ...           /* Format string and trailing args */
 ){
   va_list ap;
-  va_start(ap, zFmt);
-  char *z = sqlite3_vmprintf(zFmt, ap);
+  char *z;
   sqlite3_stmt *pRet = 0;
 
+  va_start(ap, zFmt);
+  z = sqlite3_vmprintf(zFmt, ap);
+
   if( pCheck->rc==SQLITE_OK ){
     pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0);
   }
diff --git a/manifest b/manifest
index 57d11641e5..9fac80b2d3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\nto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\nconsistent.
-D 2017-10-25T16:38:34.144
+C Fix\scompilation\sissue\s(C99-ism)\sseen\swith\sMSVC.
+D 2017-10-25T18:00:58.356
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 7941c4283bef9ecdd1b7a5c20122006b7d363836468fe7f4bef46698c55d1d93
+F ext/rtree/rtree.c 49ced71763c4bd170ceb0b46ab34c749f3356a8c0e36ce6b9e2f36dc63716609
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673
-R b2ab5ba3cbab5da53d273c6ee124bea0
-U dan
-Z 3064e6e3a20de6115706024077565624
+P dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327
+R bbf1f0574f080f1a74b750b54fc22d8d
+U mistachkin
+Z e8d8b6c6557c63f0494992168a2a41d0
diff --git a/manifest.uuid b/manifest.uuid
index d2644e0a82..b8040f3c9c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dde0bb3eab1316c3247b1755594527ca70955aab4ad4907190731f7ec092b327
\ No newline at end of file
+61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5
\ No newline at end of file

From 7e2b38c53b56b2ee025e7e0616bf9c2da3665bea Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 25 Oct 2017 18:17:24 +0000
Subject: [PATCH 216/270] Add tests cases and fix minor issues in the
 rtreecheck() function.

FossilOrigin-Name: d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
---
 ext/rtree/rtree.c         | 90 +++++++++++++++++++++++++++------------
 ext/rtree/rtreecheck.test | 49 ++++++++++++++++++++-
 manifest                  | 17 ++++----
 manifest.uuid             |  2 +-
 4 files changed, 119 insertions(+), 39 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index c7bf671c96..ff15a192a1 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -3659,7 +3659,11 @@ static sqlite3_stmt *rtreeCheckPrepare(
   z = sqlite3_vmprintf(zFmt, ap);
 
   if( pCheck->rc==SQLITE_OK ){
-    pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0);
+    if( z==0 ){
+      pCheck->rc = SQLITE_NOMEM;
+    }else{
+      pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0);
+    }
   }
 
   sqlite3_free(z);
@@ -3781,7 +3785,7 @@ static void rtreeCheckMapping(
     );
   }else if( rc==SQLITE_ROW ){
     i64 ii = sqlite3_column_int64(pStmt, 0);
-   if( ii!=iVal ){
+    if( ii!=iVal ){
       rtreeCheckAppendMsg(pCheck, 
           "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)",
           iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal
@@ -3791,10 +3795,22 @@ static void rtreeCheckMapping(
   rtreeCheckReset(pCheck, pStmt);
 }
 
+/*
+** Argument pCell points to an array of coordinates stored on an rtree page.
+** This function checks that the coordinates are internally consistent (no
+** x1>x2 conditions) and adds an error message to the RtreeCheck object
+** if they are not.
+**
+** Additionally, if pParent is not NULL, then it is assumed to point to
+** the array of coordinates on the parent page that bound the page 
+** containing pCell. In this case it is also verified that the two
+** sets of coordinates are mutually consistent and an error message added
+** to the RtreeCheck object if they are not.
+*/
 static void rtreeCheckCellCoord(
   RtreeCheck *pCheck, 
-  i64 iNode,
-  int iCell,
+  i64 iNode,                      /* Node id to use in error messages */
+  int iCell,                      /* Cell number to use in error messages */
   u8 *pCell,                      /* Pointer to cell coordinates */
   u8 *pParent                     /* Pointer to parent coordinates */
 ){
@@ -3829,6 +3845,14 @@ static void rtreeCheckCellCoord(
   }
 }
 
+/*
+** Run rtreecheck() checks on node iNode, which is at depth iDepth within
+** the r-tree structure. Argument aParent points to the array of coordinates
+** that bound node iNode on the parent node.
+**
+** If any problems are discovered, an error message is appended to the
+** report accumulated in the RtreeCheck object.
+*/
 static void rtreeCheckNode(
   RtreeCheck *pCheck,
   int iDepth,                     /* Depth of iNode (0==leaf) */
@@ -3864,19 +3888,20 @@ static void rtreeCheckNode(
             "Node %lld is too small for cell count of %d (%d bytes)", 
             iNode, nCell, nNode
         );
-      }
-      for(i=0; i<nCell; i++){
-        u8 *pCell = &aNode[4 + i*(8 + pCheck->nDim*2*4)];
-        i64 iVal = readInt64(pCell);
-        rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent);
+      }else{
+        for(i=0; i<nCell; i++){
+          u8 *pCell = &aNode[4 + i*(8 + pCheck->nDim*2*4)];
+          i64 iVal = readInt64(pCell);
+          rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent);
 
-        if( iDepth>0 ){
-          rtreeCheckMapping(pCheck, 0, iVal, iNode);
-          rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal);
-          pCheck->nNonLeaf++;
-        }else{
-          rtreeCheckMapping(pCheck, 1, iVal, iNode);
-          pCheck->nLeaf++;
+          if( iDepth>0 ){
+            rtreeCheckMapping(pCheck, 0, iVal, iNode);
+            rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal);
+            pCheck->nNonLeaf++;
+          }else{
+            rtreeCheckMapping(pCheck, 1, iVal, iNode);
+            pCheck->nLeaf++;
+          }
         }
       }
     }
@@ -3884,9 +3909,14 @@ static void rtreeCheckNode(
   }
 }
 
-static void rtreeCheckCount(
-  RtreeCheck *pCheck, const char *zTbl, i64 nExpected
-){
+/*
+** The second argument to this function must be either "_rowid" or
+** "_parent". This function checks that the number of entries in the
+** %_rowid or %_parent table is exactly nExpect. If not, it adds
+** an error message to the report in the RtreeCheck object indicated
+** by the first argument.
+*/
+static void rtreeCheckCount(RtreeCheck *pCheck, const char *zTbl, i64 nExpect){
   if( pCheck->rc==SQLITE_OK ){
     sqlite3_stmt *pCount;
     pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'",
@@ -3895,9 +3925,9 @@ static void rtreeCheckCount(
     if( pCount ){
       if( sqlite3_step(pCount)==SQLITE_ROW ){
         i64 nActual = sqlite3_column_int64(pCount, 0);
-        if( nActual!=nExpected ){
+        if( nActual!=nExpect ){
           rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table"
-              " - expected %lld, actual %lld" , zTbl, nExpected, nActual
+              " - expected %lld, actual %lld" , zTbl, nExpect, nActual
           );
         }
       }
@@ -3906,7 +3936,11 @@ static void rtreeCheckCount(
   }
 }
 
-static int rtreeCheck(
+/*
+** This function does the bulk of the work for the rtree integrity-check.
+** It is called by rtreecheck(), which is the SQL function implementation.
+*/
+static int rtreeCheckTable(
   sqlite3 *db,                    /* Database handle to access db through */
   const char *zDb,                /* Name of db ("main", "temp" etc.) */
   const char *zTab,               /* Name of rtree table to check */
@@ -3945,11 +3979,13 @@ static int rtreeCheck(
   }
 
   /* Do the actual integrity-check */
-  if( check.rc==SQLITE_OK ){
-    rtreeCheckNode(&check, 0, 0, 1);
+  if( check.nDim>=1 ){
+    if( check.rc==SQLITE_OK ){
+      rtreeCheckNode(&check, 0, 0, 1);
+    }
+    rtreeCheckCount(&check, "_rowid", check.nLeaf);
+    rtreeCheckCount(&check, "_parent", check.nNonLeaf);
   }
-  rtreeCheckCount(&check, "_rowid", check.nLeaf);
-  rtreeCheckCount(&check, "_parent", check.nNonLeaf);
 
   /* Finalize SQL statements used by the integrity-check */
   sqlite3_finalize(check.pGetNode);
@@ -4018,7 +4054,7 @@ static void rtreecheck(
     }else{
       zTab = (const char*)sqlite3_value_text(apArg[1]);
     }
-    rc = rtreeCheck(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport);
+    rc = rtreeCheckTable(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport);
     if( rc==SQLITE_OK ){
       sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT);
     }else{
diff --git a/ext/rtree/rtreecheck.test b/ext/rtree/rtreecheck.test
index eb0e8f0acb..cbe8b513c9 100644
--- a/ext/rtree/rtreecheck.test
+++ b/ext/rtree/rtreecheck.test
@@ -35,6 +35,12 @@ proc swap_int32 {blob i0 i1} {
   binary format I* $L
 }
 
+proc set_int32 {blob idx val} {
+  binary scan $blob I* L
+  lset L $idx $val
+  binary format I* $L
+}
+
 do_catchsql_test 1.0 {
   SELECT rtreecheck();
 } {1 {wrong number of arguments to function rtreecheck()}}
@@ -85,7 +91,6 @@ do_execsql_test 2.5 {
   SELECT rtreecheck('r1') 
 } {{Found (3 -> 2) in %_rowid table, expected (3 -> 1)}}
 
-################
 reset_db
 do_execsql_test 3.0 { 
   CREATE VIRTUAL TABLE r1 USING rtree_i32(id, x1, x2);
@@ -101,13 +106,53 @@ do_execsql_test 3.0 {
   SELECT rtreecheck('r1') 
 } {ok}
 
-breakpoint
 do_execsql_test 3.1 { 
   CREATE VIRTUAL TABLE r2 USING rtree_i32(id, x1, x2);
   INSERT INTO r2 VALUES(2, -1*(1<<31), -1*(1<<31)+5);
   SELECT rtreecheck('r2') 
 } {ok}
 
+do_execsql_test 3.2 {
+  BEGIN;
+    UPDATE r2_node SET data = X'123456';
+    SELECT rtreecheck('r2')!="ok";
+} {1}
+
+do_execsql_test 3.3 {
+  ROLLBACK;
+  UPDATE r2_node SET data = X'00001234';
+  SELECT rtreecheck('r2')!="ok";
+} {1}
+
+do_execsql_test 4.0 {
+  CREATE TABLE notanrtree(i);
+  SELECT rtreecheck('notanrtree');
+} {{Schema corrupt or not an rtree}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+db func set_int32 set_int32
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE r3 USING rtree_i32(id, x1, x2, y1, y2);
+  WITH x(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000
+  )
+  INSERT INTO r3 SELECT i, i, i, i, i FROM x;
+}
+do_execsql_test 5.1 {
+  BEGIN;
+    UPDATE r3_node SET data = set_int32(data, 3, 5000);
+    UPDATE r3_node SET data = set_int32(data, 4, 5000);
+    SELECT rtreecheck('r3')=='ok'
+} 0
+do_execsql_test 5.2 {
+  ROLLBACK;
+  BEGIN;
+    UPDATE r3_node SET data = set_int32(data, 3, 0);
+    UPDATE r3_node SET data = set_int32(data, 4, 0);
+    SELECT rtreecheck('r3')=='ok'
+} 0
 
 finish_test
 
diff --git a/manifest b/manifest
index 1b6fc596a8..367a1a1375 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSQL\sscalar\sfunction\srtreecheck()\sto\sthe\srtree\smodule.\sFor\srunning\schecks\sto\sensure\sthe\sshadow\stables\sused\sby\san\srtree\svirtual\stable\sare\sinternally\sconsistent.
-D 2017-10-25T18:01:58.383
+C Add\stests\scases\sand\sfix\sminor\sissues\sin\sthe\srtreecheck()\sfunction.
+D 2017-10-25T18:17:24.834
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 49ced71763c4bd170ceb0b46ab34c749f3356a8c0e36ce6b9e2f36dc63716609
+F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
@@ -349,7 +349,7 @@ F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd
 F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed
-F ext/rtree/rtreecheck.test f610cb77ca1ba611e656018a7d960cd46054baecd2f12d1149bf1fec121aa230
+F ext/rtree/rtreecheck.test 4d29103d1e16fcbf90135d1c637b833688492b063b2971dfb5dc6ba76555cfee
 F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d
 F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28
 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
@@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P adfa7ed2de3e833fff65935455e71236a59602aaf7b97ece667ab300dca9f673 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5
-R bbf1f0574f080f1a74b750b54fc22d8d
-T +closed 61870b42b033189509b244c76d28932f7a447b887ffe04a3c81461148e54cfb5
-U mistachkin
-Z cbebd749084c1ed38ad0d901e97967c8
+P 7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb
+R b18b38b834db7af5e4754d71c435762f
+U dan
+Z f91c79bfd0a67ab59612bc897b82c53b
diff --git a/manifest.uuid b/manifest.uuid
index 74306142d2..cb99ca30c6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb
\ No newline at end of file
+d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
\ No newline at end of file

From 3cd8aaa74884f2e0db7b8af83870f90e4d460837 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 25 Oct 2017 19:18:33 +0000
Subject: [PATCH 217/270] Fix the sqlite3_dbpage virtual table so that it can
 read and write from any attached database.

FossilOrigin-Name: d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
---
 manifest         |  16 ++---
 manifest.uuid    |   2 +-
 src/dbpage.c     | 169 +++++++++++++++++++++++++++++++++++------------
 test/dbpage.test |  28 +++++++-
 4 files changed, 159 insertions(+), 56 deletions(-)

diff --git a/manifest b/manifest
index 367a1a1375..a593818258 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\scases\sand\sfix\sminor\sissues\sin\sthe\srtreecheck()\sfunction.
-D 2017-10-25T18:17:24.834
+C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase.
+D 2017-10-25T19:18:33.009
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -413,7 +413,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
-F src/dbpage.c c625a0bd605d4cea9a3258b8db49a5474a04976e95a9fe380cdaf74e8eb6736d
+F src/dbpage.c 196a072dffda3ad50bb078da86bb39f1d07cabe4f88e2b12482f4f5ffce73352
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304
@@ -715,7 +715,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b
 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
-F test/dbpage.test 10b9e91d07b0892444fff4578706648e955b5fb260218298f838da74f0d9d211
+F test/dbpage.test ace7a8d2d66b0ad043dacc61d206991184c9916de6862bf4ff165a7906a624af
 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7d26498063bbc4525976ace698fafcf161d939908f4ffb36d7a4df8fb1bce1fb
-R b18b38b834db7af5e4754d71c435762f
-U dan
-Z f91c79bfd0a67ab59612bc897b82c53b
+P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
+R de06ad2d264283eba96aba95f64ee6f5
+U drh
+Z 3b4d77a61659a96790923a30bda00b59
diff --git a/manifest.uuid b/manifest.uuid
index cb99ca30c6..ce16a7caf0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
\ No newline at end of file
+d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
index d21c5b6df1..54fb5beaa7 100644
--- a/src/dbpage.c
+++ b/src/dbpage.c
@@ -42,17 +42,24 @@ struct DbpageCursor {
   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
   int pgno;                       /* Current page number */
   int mxPgno;                     /* Last page to visit on this scan */
+  Pager *pPager;                  /* Pager being read/written */
+  DbPage *pPage1;                 /* Page 1 of the database */
+  int iDb;                        /* Index of database to analyze */
+  int szPage;                     /* Size of each page in bytes */
 };
 
 struct DbpageTable {
   sqlite3_vtab base;              /* Base class.  Must be first */
   sqlite3 *db;                    /* The database */
-  Pager *pPager;                  /* Pager being read/written */
-  int iDb;                        /* Index of database to analyze */
-  int szPage;                     /* Size of each page in bytes */
-  int nPage;                      /* Number of pages in the file */
 };
 
+/* Columns */
+#define DBPAGE_COLUMN_PGNO    0
+#define DBPAGE_COLUMN_DATA    1
+#define DBPAGE_COLUMN_SCHEMA  2
+
+
+
 /*
 ** Connect to or create a dbpagevfs virtual table.
 */
@@ -65,19 +72,7 @@ static int dbpageConnect(
 ){
   DbpageTable *pTab = 0;
   int rc = SQLITE_OK;
-  int iDb;
 
-  if( argc>=4 ){
-    Token nm;
-    sqlite3TokenInit(&nm, (char*)argv[3]);
-    iDb = sqlite3FindDb(db, &nm);
-    if( iDb<0 ){
-      *pzErr = sqlite3_mprintf("no such schema: %s", argv[3]);
-      return SQLITE_ERROR;
-    }
-  }else{
-    iDb = 0;
-  }
   rc = sqlite3_declare_vtab(db, 
           "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)");
   if( rc==SQLITE_OK ){
@@ -87,11 +82,8 @@ static int dbpageConnect(
 
   assert( rc==SQLITE_OK || pTab==0 );
   if( rc==SQLITE_OK ){
-    Btree *pBt = db->aDb[iDb].pBt;
     memset(pTab, 0, sizeof(DbpageTable));
     pTab->db = db;
-    pTab->iDb = iDb;
-    pTab->pPager = pBt ? sqlite3BtreePager(pBt) : 0;
   }
 
   *ppVtab = (sqlite3_vtab*)pTab;
@@ -109,24 +101,55 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){
 /*
 ** idxNum:
 **
-**     0     full table scan
-**     1     pgno=?1
+**     0     schema=main, full table scan
+**     1     schema=main, pgno=?1
+**     2     schema=?1, full table scan
+**     3     schema=?1, pgno=?2
 */
 static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
   int i;
-  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
+  int iPlan = 0;
+
+  /* If there is a schema= constraint, it must be honored.  Report a
+  ** ridiculously large estimated cost if the schema= constraint is
+  ** unavailable
+  */
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+    if( p->iColumn!=DBPAGE_COLUMN_SCHEMA ) continue;
+    if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( !p->usable ){
+      /* No solution.  Use the default SQLITE_BIG_DBL cost */
+      pIdxInfo->estimatedRows = 0x7fffffff;
+      return SQLITE_OK;
+    }
+    iPlan = 2;
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    break;
+  }
+
+  /* If we reach this point, it means that either there is no schema=
+  ** constraint (in which case we use the "main" schema) or else the
+  ** schema constraint was accepted.  Lower the estimated cost accordingly
+  */
+  pIdxInfo->estimatedCost = 1.0e6;
+
+  /* Check for constraints against pgno */
   for(i=0; i<pIdxInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
     if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
       pIdxInfo->estimatedRows = 1;
       pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE;
       pIdxInfo->estimatedCost = 1.0;
-      pIdxInfo->idxNum = 1;
-      pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = iPlan ? 2 : 1;
       pIdxInfo->aConstraintUsage[i].omit = 1;
+      iPlan |= 1;
       break;
     }
   }
+  pIdxInfo->idxNum = iPlan;
+
   if( pIdxInfo->nOrderBy>=1
    && pIdxInfo->aOrderBy[0].iColumn<=0
    && pIdxInfo->aOrderBy[0].desc==0
@@ -160,6 +183,7 @@ static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
 */
 static int dbpageClose(sqlite3_vtab_cursor *pCursor){
   DbpageCursor *pCsr = (DbpageCursor *)pCursor;
+  if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1);
   sqlite3_free(pCsr);
   return SQLITE_OK;
 }
@@ -179,6 +203,16 @@ static int dbpageEof(sqlite3_vtab_cursor *pCursor){
   return pCsr->pgno > pCsr->mxPgno;
 }
 
+/*
+** idxNum:
+**
+**     0     schema=main, full table scan
+**     1     schema=main, pgno=?1
+**     2     schema=?1, full table scan
+**     3     schema=?1, pgno=?2
+**
+** idxStr is not used
+*/
 static int dbpageFilter(
   sqlite3_vtab_cursor *pCursor, 
   int idxNum, const char *idxStr,
@@ -187,22 +221,40 @@ static int dbpageFilter(
   DbpageCursor *pCsr = (DbpageCursor *)pCursor;
   DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
   int rc = SQLITE_OK;
-  Btree *pBt = pTab->db->aDb[pTab->iDb].pBt;
+  sqlite3 *db = pTab->db;
+  Btree *pBt;
 
-  pTab->szPage = sqlite3BtreeGetPageSize(pBt);
-  pTab->nPage = sqlite3BtreeLastPage(pBt);
-  if( idxNum==1 ){
-    pCsr->pgno = sqlite3_value_int(argv[0]);
-    if( pCsr->pgno<1 || pCsr->pgno>pTab->nPage ){
+  /* Default setting is no rows of result */
+  pCsr->pgno = 1; 
+  pCsr->mxPgno = 0;
+
+  if( idxNum & 2 ){
+    const char *zSchema;
+    assert( argc>=1 );
+    zSchema = (const char*)sqlite3_value_text(argv[0]);
+    pCsr->iDb = sqlite3FindDbName(db, zSchema);
+    if( pCsr->iDb<0 ) return SQLITE_OK;
+  }else{
+    pCsr->iDb = 0;
+  }
+  pBt = db->aDb[pCsr->iDb].pBt;
+  if( pBt==0 ) return SQLITE_OK;
+  pCsr->pPager = sqlite3BtreePager(pBt);
+  pCsr->szPage = sqlite3BtreeGetPageSize(pBt);
+  pCsr->mxPgno = sqlite3BtreeLastPage(pBt);
+  if( idxNum & 1 ){
+    assert( argc>(idxNum>>1) );
+    pCsr->pgno = sqlite3_value_int(argv[idxNum>>1]);
+    if( pCsr->pgno<1 || pCsr->pgno>pCsr->mxPgno ){
       pCsr->pgno = 1;
       pCsr->mxPgno = 0;
     }else{
       pCsr->mxPgno = pCsr->pgno;
     }
   }else{
-    pCsr->pgno = 1;
-    pCsr->mxPgno = pTab->nPage;
+    assert( pCsr->pgno==1 );
   }
+  rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0);
   return rc;
 }
 
@@ -212,7 +264,6 @@ static int dbpageColumn(
   int i
 ){
   DbpageCursor *pCsr = (DbpageCursor *)pCursor;
-  DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
   int rc = SQLITE_OK;
   switch( i ){
     case 0: {           /* pgno */
@@ -221,9 +272,9 @@ static int dbpageColumn(
     }
     case 1: {           /* data */
       DbPage *pDbPage = 0;
-      rc = sqlite3PagerGet(pTab->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0);
+      rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0);
       if( rc==SQLITE_OK ){
-        sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pTab->szPage,
+        sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage,
                             SQLITE_TRANSIENT);
       }
       sqlite3PagerUnref(pDbPage);
@@ -231,7 +282,7 @@ static int dbpageColumn(
     }
     default: {          /* schema */
       sqlite3 *db = sqlite3_context_db_handle(ctx);
-      sqlite3_result_text(ctx, db->aDb[pTab->iDb].zDbSName, -1, SQLITE_STATIC);
+      sqlite3_result_text(ctx, db->aDb[pCsr->iDb].zDbSName, -1, SQLITE_STATIC);
       break;
     }
   }
@@ -255,33 +306,47 @@ static int dbpageUpdate(
   DbPage *pDbPage = 0;
   int rc = SQLITE_OK;
   char *zErr = 0;
+  const char *zSchema;
+  int iDb;
+  Btree *pBt;
+  Pager *pPager;
+  int szPage;
 
   if( argc==1 ){
     zErr = "cannot delete";
     goto update_fail;
   }
   pgno = sqlite3_value_int(argv[0]);
-  if( pgno<1 || pgno>pTab->nPage ){
-    zErr = "bad page number";
-    goto update_fail;
-  }
   if( sqlite3_value_int(argv[1])!=pgno ){
     zErr = "cannot insert";
     goto update_fail;
   }
+  zSchema = (const char*)sqlite3_value_text(argv[4]);
+  iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1;
+  if( iDb<0 ){
+    zErr = "no such schema";
+    goto update_fail;
+  }
+  pBt = pTab->db->aDb[iDb].pBt;
+  if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){
+    zErr = "bad page number";
+    goto update_fail;
+  }
+  szPage = sqlite3BtreeGetPageSize(pBt);
   if( sqlite3_value_type(argv[3])!=SQLITE_BLOB 
-   || sqlite3_value_bytes(argv[3])!=pTab->szPage 
+   || sqlite3_value_bytes(argv[3])!=szPage
   ){
     zErr = "bad page value";
     goto update_fail;
   }
-  rc = sqlite3PagerGet(pTab->pPager, pgno, (DbPage**)&pDbPage, 0);
+  pPager = sqlite3BtreePager(pBt);
+  rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0);
   if( rc==SQLITE_OK ){
     rc = sqlite3PagerWrite(pDbPage);
     if( rc==SQLITE_OK ){
       memcpy(sqlite3PagerGetData(pDbPage),
              sqlite3_value_blob(argv[3]),
-             pTab->szPage);
+             szPage);
     }
   }
   sqlite3PagerUnref(pDbPage);
@@ -293,6 +358,22 @@ update_fail:
   return SQLITE_ERROR;
 }
 
+/* Since we do not know in advance which database files will be
+** written by the sqlite_dbpage virtual table, start a write transaction
+** on them all.
+*/
+static int dbpageBegin(sqlite3_vtab *pVtab){
+  DbpageTable *pTab = (DbpageTable *)pVtab;
+  sqlite3 *db = pTab->db;
+  int i;
+  for(i=0; i<db->nDb; i++){
+    Btree *pBt = db->aDb[i].pBt;
+    if( pBt ) sqlite3BtreeBeginTrans(pBt, 1);
+  }
+  return SQLITE_OK;
+}
+
+
 /*
 ** Invoke this routine to register the "dbpage" virtual table module
 */
@@ -312,7 +393,7 @@ int sqlite3DbpageRegister(sqlite3 *db){
     dbpageColumn,                 /* xColumn - read data */
     dbpageRowid,                  /* xRowid - read data */
     dbpageUpdate,                 /* xUpdate */
-    0,                            /* xBegin */
+    dbpageBegin,                  /* xBegin */
     0,                            /* xSync */
     0,                            /* xCommit */
     0,                            /* xRollback */
diff --git a/test/dbpage.test b/test/dbpage.test
index a8ef8c0800..5a68d3caba 100644
--- a/test/dbpage.test
+++ b/test/dbpage.test
@@ -49,6 +49,13 @@ do_execsql_test 140 {
 do_execsql_test 150 {
   SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage WHERE pgno=0;
 } {}
+do_execsql_test 160 {
+  ATTACH ':memory:' AS aux1;
+  PRAGMA aux1.page_size=4096;
+  CREATE TABLE aux1.t2(a,b,c);
+  INSERT INTO t2 VALUES(11,12,13);
+  SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('aux1');
+} {1 X'53514C6974' 2 X'0D00000001'}
 
 do_execsql_test 200 {
   CREATE TEMP TABLE saved_content(x);
@@ -67,8 +74,23 @@ do_execsql_test 230 {
 do_catchsql_test 230 {
   PRAGMA integrity_check;
 } {0 ok}
-
-
-
+do_execsql_test 240 {
+  DELETE FROM saved_content;
+  INSERT INTO saved_content(x) 
+     SELECT data FROM sqlite_dbpage WHERE schema='aux1' AND pgno=2;
+} {}
+do_execsql_test 241 {
+  UPDATE sqlite_dbpage SET data=zeroblob(4096) WHERE pgno=2 AND schema='aux1';
+} {}
+do_catchsql_test 250 {
+  PRAGMA aux1.integrity_check;
+} {1 {database disk image is malformed}}
+do_execsql_test 260 {
+  UPDATE sqlite_dbpage SET data=(SELECT x FROM saved_content)
+   WHERE pgno=2 AND schema='aux1';
+} {}
+do_catchsql_test 270 {
+  PRAGMA aux1.integrity_check;
+} {0 ok}
 
 finish_test

From 4544e28425a02ee874e12e36ba7f0f9dfbf9c4b6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 26 Oct 2017 15:21:56 +0000
Subject: [PATCH 218/270] Fix a bug causing "make amalgmation-autoconf" to
 create an invalid package if either the current or parent directory contains
 a file named install.sh or install-sh.

FossilOrigin-Name: e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d
---
 autoconf/configure.ac |  1 +
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/autoconf/configure.ac b/autoconf/configure.ac
index 5a607de054..c207bdf74d 100644
--- a/autoconf/configure.ac
+++ b/autoconf/configure.ac
@@ -12,6 +12,7 @@
 AC_PREREQ(2.61)
 AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
 AC_CONFIG_SRCDIR([sqlite3.c])
+AC_CONFIG_AUX_DIR([.])
 
 # Use automake.
 AM_INIT_AUTOMAKE([foreign])
diff --git a/manifest b/manifest
index a593818258..03eafd6f57 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase.
-D 2017-10-25T19:18:33.009
+C Fix\sa\sbug\scausing\s"make\samalgmation-autoconf"\sto\screate\san\sinvalid\spackage\sif\neither\sthe\scurrent\sor\sparent\sdirectory\scontains\sa\sfile\snamed\sinstall.sh\sor\ninstall-sh.
+D 2017-10-26T15:21:56.963
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -14,7 +14,7 @@ F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6
 F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578
+F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
-R de06ad2d264283eba96aba95f64ee6f5
-U drh
-Z 3b4d77a61659a96790923a30bda00b59
+P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
+R b4e850a770e92ea272826b571ec0824a
+U dan
+Z d2098b60d98f2c55ca7e6fbf02761c40
diff --git a/manifest.uuid b/manifest.uuid
index ce16a7caf0..8c0d2fd56c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
\ No newline at end of file
+e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d
\ No newline at end of file

From 7ea11066b62c0dac40ed249d1eb64d6f2b9e01e3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 26 Oct 2017 18:43:19 +0000
Subject: [PATCH 219/270] Extra ".selecttrace" output following aggregate
 analysis.  No changes to production builds.

FossilOrigin-Name: 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/select.c  | 20 +++++++++++++++++++-
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 03eafd6f57..8c7e2d8801 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbug\scausing\s"make\samalgmation-autoconf"\sto\screate\san\sinvalid\spackage\sif\neither\sthe\scurrent\sor\sparent\sdirectory\scontains\sa\sfile\snamed\sinstall.sh\sor\ninstall-sh.
-D 2017-10-26T15:21:56.963
+C Extra\s".selecttrace"\soutput\sfollowing\saggregate\sanalysis.\s\sNo\schanges\sto\nproduction\sbuilds.
+D 2017-10-26T18:43:19.787
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -463,7 +463,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8
+F src/select.c d055dc3a428ab2dd94ffdbaef50d90b41579be8ffe258591615d3d83bd859f7b
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
-R b4e850a770e92ea272826b571ec0824a
-U dan
-Z d2098b60d98f2c55ca7e6fbf02761c40
+P e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d
+R d61bdf0ef8749a81bf4548782da0df22
+U drh
+Z eef6957fcaef94fa72ebc9ad261c03d1
diff --git a/manifest.uuid b/manifest.uuid
index 8c0d2fd56c..0db876cf23 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d
\ No newline at end of file
+6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 0e2328120b..0f73c30f64 100644
--- a/src/select.c
+++ b/src/select.c
@@ -5659,6 +5659,24 @@ int sqlite3Select(
     }
     sAggInfo.mxReg = pParse->nMem;
     if( db->mallocFailed ) goto select_end;
+#if SELECTTRACE_ENABLED
+    if( sqlite3SelectTrace & 0x400 ){
+      int ii;
+      SELECTTRACE(0x400,pParse,p,("After aggregate analysis:\n"));
+      sqlite3TreeViewSelect(0, p, 0);
+      for(ii=0; ii<sAggInfo.nColumn; ii++){
+        sqlite3DebugPrintf("agg-column[%d] iMem=%d\n",
+            ii, sAggInfo.aCol[ii].iMem);
+        sqlite3TreeViewExpr(0, sAggInfo.aCol[ii].pExpr, 0);
+      }
+      for(ii=0; ii<sAggInfo.nFunc; ii++){
+        sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
+            ii, sAggInfo.aFunc[ii].iMem);
+        sqlite3TreeViewExpr(0, sAggInfo.aFunc[ii].pExpr, 0);
+      }
+    }
+#endif
+
 
     /* Processing for aggregates with GROUP BY is very different and
     ** much more complex than aggregates without a GROUP BY.
@@ -5956,7 +5974,7 @@ int sqlite3Select(
         **   SELECT max(x) FROM ...
         **
         ** If it is, then ask the code in where.c to attempt to sort results
-        ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. 
+        ** as if there was an "ORDER BY x" or "ORDER BY x DESC" clause. 
         ** If where.c is able to produce results sorted in this order, then
         ** add vdbe code to break out of the processing loop after the 
         ** first iteration (since the first iteration of the loop is 

From 47d9f83982e5996134fa1f4c744ff3192f1003f2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 26 Oct 2017 20:04:28 +0000
Subject: [PATCH 220/270] Enhance the min/max optimization so that it works
 with indexes on expressions.

FossilOrigin-Name: 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833
---
 manifest          |  14 ++---
 manifest.uuid     |   2 +-
 src/select.c      | 135 ++++++++++++++++++----------------------------
 test/minmax2.test |  34 ++++++++++++
 4 files changed, 95 insertions(+), 90 deletions(-)

diff --git a/manifest b/manifest
index 8c7e2d8801..f1127b40df 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Extra\s".selecttrace"\soutput\sfollowing\saggregate\sanalysis.\s\sNo\schanges\sto\nproduction\sbuilds.
-D 2017-10-26T18:43:19.787
+C Enhance\sthe\smin/max\soptimization\sso\sthat\sit\sworks\swith\sindexes\son\nexpressions.
+D 2017-10-26T20:04:28.629
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -463,7 +463,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c d055dc3a428ab2dd94ffdbaef50d90b41579be8ffe258591615d3d83bd859f7b
+F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1044,7 +1044,7 @@ F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e
 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08
 F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41
-F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
+F test/minmax2.test dae92964ac87c1d2ef978c582e81a95e11c00f1cbef68980bfb2abaf10315063
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
 F test/misc1.test 76737c259537586355f45e2a1e121b6e91b5476c4604ad5c53d1abfcb3acf786
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e1faa7859451ee969f0ce02d2d4e4c54d0e2e018b37b8faf690e6b87593cce7d
-R d61bdf0ef8749a81bf4548782da0df22
+P 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76
+R c17176b8dc9ac053e5191ecbda77db6f
 U drh
-Z eef6957fcaef94fa72ebc9ad261c03d1
+Z 1bd9fa3ff01b6d52accac3363bceb595
diff --git a/manifest.uuid b/manifest.uuid
index 0db876cf23..41f1ba3965 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76
\ No newline at end of file
+6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 0f73c30f64..69ef06aeb5 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3919,42 +3919,44 @@ static int pushDownWhereTerms(
 #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
 
 /*
-** Based on the contents of the AggInfo structure indicated by the first
-** argument, this function checks if the following are true:
+** The pFunc is the only aggregate function in the query.  Check to see
+** if the query is a candidate for the min/max optimization. 
 **
-**    * the query contains just a single aggregate function,
-**    * the aggregate function is either min() or max(), and
-**    * the argument to the aggregate function is a column value.
+** If the query is a candidate for the min/max optimization, then set
+** *ppMinMax to be an ORDER BY clause to be used for the optimization
+** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on
+** whether pFunc is a min() or max() function.
 **
-** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX
-** is returned as appropriate. Also, *ppMinMax is set to point to the 
-** list of arguments passed to the aggregate before returning.
+** If the query is not a candidate for the min/max optimization, return
+** WHERE_ORDERBY_NORMAL (which must be zero).
 **
-** Or, if the conditions above are not met, *ppMinMax is set to 0 and
-** WHERE_ORDERBY_NORMAL is returned.
+** This routine must be called after aggregate functions have been
+** located but before their arguments have been subjected to aggregate
+** analysis.
 */
-static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){
-  int eRet = WHERE_ORDERBY_NORMAL;          /* Return value */
+static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){
+  int eRet = WHERE_ORDERBY_NORMAL;      /* Return value */
+  ExprList *pEList = pFunc->x.pList;    /* Arguments to agg function */
+  const char *zFunc;                    /* Name of aggregate function pFunc */
+  ExprList *pOrderBy;
+  u8 sortOrder;
 
-  *ppMinMax = 0;
-  if( pAggInfo->nFunc==1 ){
-    Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */
-    ExprList *pEList = pExpr->x.pList;      /* Arguments to agg function */
-
-    assert( pExpr->op==TK_AGG_FUNCTION );
-    if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){
-      const char *zFunc = pExpr->u.zToken;
-      if( sqlite3StrICmp(zFunc, "min")==0 ){
-        eRet = WHERE_ORDERBY_MIN;
-        *ppMinMax = pEList;
-      }else if( sqlite3StrICmp(zFunc, "max")==0 ){
-        eRet = WHERE_ORDERBY_MAX;
-        *ppMinMax = pEList;
-      }
-    }
+  assert( *ppMinMax==0 );
+  assert( pFunc->op==TK_AGG_FUNCTION );
+  if( pEList==0 || pEList->nExpr!=1 ) return eRet;
+  zFunc = pFunc->u.zToken;
+  if( sqlite3StrICmp(zFunc, "min")==0 ){
+    eRet = WHERE_ORDERBY_MIN;
+    sortOrder = SQLITE_SO_ASC;
+  }else if( sqlite3StrICmp(zFunc, "max")==0 ){
+    eRet = WHERE_ORDERBY_MAX;
+    sortOrder = SQLITE_SO_DESC;
+  }else{
+    return eRet;
   }
-
-  assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 );
+  *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0);
+  assert( pOrderBy!=0 || db->mallocFailed );
+  if( pOrderBy ) pOrderBy->a[0].sortOrder = sortOrder;
   return eRet;
 }
 
@@ -5135,6 +5137,8 @@ int sqlite3Select(
   AggInfo sAggInfo;      /* Information used by aggregate queries */
   int iEnd;              /* Address of the end of the query */
   sqlite3 *db;           /* The database connection */
+  ExprList *pMinMaxOrderBy = 0;  /* Added ORDER BY for min/max queries */
+  u8 minMaxFlag;                 /* Flag for min/max queries */
 
 #ifndef SQLITE_OMIT_EXPLAIN
   int iRestoreSelectId = pParse->iSelectId;
@@ -5651,6 +5655,11 @@ int sqlite3Select(
       sqlite3ExprAnalyzeAggregates(&sNC, pHaving);
     }
     sAggInfo.nAccumulator = sAggInfo.nColumn;
+    if( p->pGroupBy==0 && p->pHaving==0 && sAggInfo.nFunc==1 ){
+      minMaxFlag = minMaxQuery(db, sAggInfo.aFunc[0].pExpr, &pMinMaxOrderBy);
+    }else{
+      minMaxFlag = WHERE_ORDERBY_NORMAL;
+    }
     for(i=0; i<sAggInfo.nFunc; i++){
       assert( !ExprHasProperty(sAggInfo.aFunc[i].pExpr, EP_xIsSelect) );
       sNC.ncFlags |= NC_InAggFunc;
@@ -5906,7 +5915,6 @@ int sqlite3Select(
      
     } /* endif pGroupBy.  Begin aggregate queries without GROUP BY: */
     else {
-      ExprList *pDel = 0;
 #ifndef SQLITE_OMIT_BTREECOUNT
       Table *pTab;
       if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){
@@ -5968,67 +5976,31 @@ int sqlite3Select(
       }else
 #endif /* SQLITE_OMIT_BTREECOUNT */
       {
-        /* Check if the query is of one of the following forms:
-        **
-        **   SELECT min(x) FROM ...
-        **   SELECT max(x) FROM ...
-        **
-        ** If it is, then ask the code in where.c to attempt to sort results
-        ** as if there was an "ORDER BY x" or "ORDER BY x DESC" clause. 
-        ** If where.c is able to produce results sorted in this order, then
-        ** add vdbe code to break out of the processing loop after the 
-        ** first iteration (since the first iteration of the loop is 
-        ** guaranteed to operate on the row with the minimum or maximum 
-        ** value of x, the only row required).
-        **
-        ** A special flag must be passed to sqlite3WhereBegin() to slightly
-        ** modify behavior as follows:
-        **
-        **   + If the query is a "SELECT min(x)", then the loop coded by
-        **     where.c should not iterate over any values with a NULL value
-        **     for x.
-        **
-        **   + The optimizer code in where.c (the thing that decides which
-        **     index or indices to use) should place a different priority on 
-        **     satisfying the 'ORDER BY' clause than it does in other cases.
-        **     Refer to code and comments in where.c for details.
-        */
-        ExprList *pMinMax = 0;
-        u8 flag = WHERE_ORDERBY_NORMAL;
-        
-        assert( p->pGroupBy==0 );
-        assert( flag==0 );
-        if( p->pHaving==0 ){
-          flag = minMaxQuery(&sAggInfo, &pMinMax);
-        }
-        assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) );
-
-        if( flag ){
-          pMinMax = sqlite3ExprListDup(db, pMinMax, 0);
-          pDel = pMinMax;
-          assert( db->mallocFailed || pMinMax!=0 );
-          if( !db->mallocFailed ){
-            pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0;
-            pMinMax->a[0].pExpr->op = TK_COLUMN;
-          }
-        }
-  
         /* This case runs if the aggregate has no GROUP BY clause.  The
         ** processing is much simpler since there is only a single row
         ** of output.
         */
+        assert( p->pGroupBy==0 );
         resetAccumulator(pParse, &sAggInfo);
-        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0);
+
+        /* If this query is a candidate for the min/max optimization, then
+        ** minMaxFlag will have been previously set to either
+        ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will
+        ** be an appropriate ORDER BY expression for the optimization.
+        */
+        assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 );
+        assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 );
+
+        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
+                                   0, minMaxFlag, 0);
         if( pWInfo==0 ){
-          sqlite3ExprListDelete(db, pDel);
           goto select_end;
         }
         updateAccumulator(pParse, &sAggInfo);
-        assert( pMinMax==0 || pMinMax->nExpr==1 );
         if( sqlite3WhereIsOrdered(pWInfo)>0 ){
           sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo));
           VdbeComment((v, "%s() by index",
-                (flag==WHERE_ORDERBY_MIN?"min":"max")));
+                (minMaxFlag==WHERE_ORDERBY_MIN?"min":"max")));
         }
         sqlite3WhereEnd(pWInfo);
         finalizeAggFunctions(pParse, &sAggInfo);
@@ -6038,7 +6010,6 @@ int sqlite3Select(
       sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL);
       selectInnerLoop(pParse, p, -1, 0, 0, 
                       pDest, addrEnd, addrEnd);
-      sqlite3ExprListDelete(db, pDel);
     }
     sqlite3VdbeResolveLabel(v, addrEnd);
     
@@ -6070,7 +6041,7 @@ int sqlite3Select(
   */
 select_end:
   explainSetInteger(pParse->iSelectId, iRestoreSelectId);
-
+  sqlite3ExprListDelete(db, pMinMaxOrderBy);
   sqlite3DbFree(db, sAggInfo.aCol);
   sqlite3DbFree(db, sAggInfo.aFunc);
 #if SELECTTRACE_ENABLED
diff --git a/test/minmax2.test b/test/minmax2.test
index da8fec30ce..b6114f2e51 100644
--- a/test/minmax2.test
+++ b/test/minmax2.test
@@ -383,5 +383,39 @@ do_test minmax2-10.12 {
   }
 } {{} {}}
 
+# 2017-10-26.  Extend the min/max optimization to indexes on expressions
+#
+do_execsql_test minmax2-11.100 {
+  CREATE TABLE t11(a,b,c);
+  INSERT INTO t11(a,b,c) VALUES(1,10,5),(2,8,11),(3,1,4),(4,20,1),(5,16,4);
+  CREATE INDEX t11bc ON t11(b+c);
+  SELECT max(b+c) FROM t11;
+} {21}
+do_execsql_test minmax2-11.110 {
+  SELECT a, max(b+c) FROM t11;
+} {4 21}
+do_test minmax2-11.111 {
+  db eval {SELECT max(b+c) FROM t11}
+  db status step
+} {0}
+do_test minmax2-11.112 {
+  db eval {SELECT max(c+b) FROM t11}
+  db status step
+} {4}
+do_execsql_test minmax2-11.120 {
+  SELECT a, min(b+c) FROM t11;
+} {3 5}
+do_test minmax2-11.121 {
+  db eval {SELECT min(b+c) FROM t11}
+  db status step
+} {0}
+do_test minmax2-11.122 {
+  db eval {SELECT min(c+b) FROM t11}
+  db status step
+} {4}
+do_execsql_test minmax2-11.130 {
+  INSERT INTO t11(a,b,c) VALUES(6,NULL,0),(7,0,NULL);
+  SELECT a, min(b+c) FROM t11;
+} {3 5}
 
 finish_test

From 6e61c5f895d77cba99c54c6b5b758f423e25c580 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 27 Oct 2017 12:27:25 +0000
Subject: [PATCH 221/270] Additional test case demonstrating the ability to
 access structure elements using quoted names in the JSON1 extension.

FossilOrigin-Name: 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
---
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 test/json101.test | 40 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index f1127b40df..9f32f230d3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\smin/max\soptimization\sso\sthat\sit\sworks\swith\sindexes\son\nexpressions.
-D 2017-10-26T20:04:28.629
+C Additional\stest\scase\sdemonstrating\sthe\sability\sto\saccess\sstructure\selements\nusing\squoted\snames\sin\sthe\sJSON1\sextension.
+D 2017-10-27T12:27:25.402
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -988,7 +988,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307
 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa
 F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json101.test 2bd3703a8566e39f8efee8b83bb60aac363fa312e69c94b4024114e0fe77e380
+F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095
 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6fbf74ab3189b9cb20cf7cc8db0226eb935a8eab477cc83b8390492fcf2f1f76
-R c17176b8dc9ac053e5191ecbda77db6f
+P 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833
+R 8ebb970e40e0966c50eb4f87e0c6dbf4
 U drh
-Z 1bd9fa3ff01b6d52accac3363bceb595
+Z eb1c1a1b07b761c18b2113b5597a4235
diff --git a/manifest.uuid b/manifest.uuid
index 41f1ba3965..f88c90a99a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833
\ No newline at end of file
+8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
\ No newline at end of file
diff --git a/test/json101.test b/test/json101.test
index 7b0292ad03..4bfcc2d80d 100644
--- a/test/json101.test
+++ b/test/json101.test
@@ -722,4 +722,44 @@ do_execsql_test json-11.3 {
   /* } */
 } {0}
 
+# 2017-10-27.  Demonstrate the ability to access an element from
+# a json structure even though the element name constains a "."
+# character, by quoting the element name in the path.
+#
+do_execsql_test json-12.100 {
+  CREATE TABLE t12(x);
+  INSERT INTO t12(x) VALUES(
+    '{"settings":
+        {"layer2":
+           {"hapax.legomenon":
+              {"forceDisplay":true,
+               "transliterate":true,
+               "add.footnote":true,
+               "summary.report":true},
+            "dis.legomenon":
+              {"forceDisplay":true,
+               "transliterate":false,
+               "add.footnote":false,
+               "summary.report":true},
+            "tris.legomenon":
+              {"forceDisplay":true,
+               "transliterate":false,
+               "add.footnote":false,
+               "summary.report":false}
+           }
+        }
+     }');
+} {}
+do_execsql_test json-12.110 {
+  SELECT json_remove(x, '$.settings.layer2."dis.legomenon".forceDisplay')
+    FROM t12;
+} {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}}
+do_execsql_test json-12.120 {
+  SELECT json_extract(x, '$.settings.layer2."tris.legomenon"."summary.report"')
+    FROM t12;
+} {0}
+
+
+
+
 finish_test

From b4bab90b31afa3c4646dbae5f650b2b17cb64a69 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 27 Oct 2017 17:09:44 +0000
Subject: [PATCH 222/270] Handle the Ctrl-C event for the shell on Win32.

FossilOrigin-Name: c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 17 ++++++++++++++++-
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 9f32f230d3..2d13f91b8d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Additional\stest\scase\sdemonstrating\sthe\sability\sto\saccess\sstructure\selements\nusing\squoted\snames\sin\sthe\sJSON1\sextension.
-D 2017-10-27T12:27:25.402
+C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32.
+D 2017-10-27T17:09:44.546
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -464,7 +464,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11
-F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
+F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6caec9082b3ad6d3f89f6291084a8f5c80f296630e2e94bc764389ac1a47a833
-R 8ebb970e40e0966c50eb4f87e0c6dbf4
-U drh
-Z eb1c1a1b07b761c18b2113b5597a4235
+P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
+R b04098ed5c4b7451669ce8c4fddf24e2
+U mistachkin
+Z 04b0a33e7f8b0430861d645ce26e5f19
diff --git a/manifest.uuid b/manifest.uuid
index f88c90a99a..478c761165 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
\ No newline at end of file
+c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 8f5ed59e93..e85ebd3c93 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1205,7 +1205,6 @@ static void output_csv(ShellState *p, const char *z, int bSep){
   }
 }
 
-#ifdef SIGINT
 /*
 ** This routine runs when the user presses Ctrl-C
 */
@@ -1215,6 +1214,20 @@ static void interrupt_handler(int NotUsed){
   if( seenInterrupt>2 ) exit(1);
   if( globalDb ) sqlite3_interrupt(globalDb);
 }
+
+#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
+/*
+** This routine runs for console events (e.g. Ctrl-C) on Win32
+*/
+static BOOL WINAPI ConsoleCtrlHandler(
+  DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */
+){
+  if( dwCtrlType==CTRL_C_EVENT ){
+    interrupt_handler(0);
+    return TRUE;
+  }
+  return FALSE;
+}
 #endif
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
@@ -6689,6 +6702,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   */
 #ifdef SIGINT
   signal(SIGINT, interrupt_handler);
+#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
+  SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE);
 #endif
 
 #ifdef SQLITE_SHELL_DBNAME_PROC

From 0503f2acfa16288c28a3d13d87f06ef7c65da19d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 27 Oct 2017 18:24:11 +0000
Subject: [PATCH 223/270] Fix a Pager ref-count leak in the sqlite_dbpage
 virtual table.

FossilOrigin-Name: c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/dbpage.c     |  3 ++-
 test/dbpage.test |  8 ++++++++
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 2d13f91b8d..7abf007c96 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32.
-D 2017-10-27T17:09:44.546
+C Fix\sa\sPager\sref-count\sleak\sin\sthe\ssqlite_dbpage\svirtual\stable.
+D 2017-10-27T18:24:11.866
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -413,7 +413,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
-F src/dbpage.c 196a072dffda3ad50bb078da86bb39f1d07cabe4f88e2b12482f4f5ffce73352
+F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304
@@ -715,7 +715,7 @@ F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b
 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
-F test/dbpage.test ace7a8d2d66b0ad043dacc61d206991184c9916de6862bf4ff165a7906a624af
+F test/dbpage.test dbf50a4d361f9e45a979432c727506065113124478a7d2db12074fa655e65d6c
 F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5
 F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
-R b04098ed5c4b7451669ce8c4fddf24e2
-U mistachkin
-Z 04b0a33e7f8b0430861d645ce26e5f19
+P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
+R f85ba3328211fce31f6fd8c8e66f1d27
+U drh
+Z a44af1749e0d20716f81debef1ff6db8
diff --git a/manifest.uuid b/manifest.uuid
index 478c761165..f8e9e5b80a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
\ No newline at end of file
+c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
index 54fb5beaa7..f287e72229 100644
--- a/src/dbpage.c
+++ b/src/dbpage.c
@@ -220,7 +220,7 @@ static int dbpageFilter(
 ){
   DbpageCursor *pCsr = (DbpageCursor *)pCursor;
   DbpageTable *pTab = (DbpageTable *)pCursor->pVtab;
-  int rc = SQLITE_OK;
+  int rc;
   sqlite3 *db = pTab->db;
   Btree *pBt;
 
@@ -254,6 +254,7 @@ static int dbpageFilter(
   }else{
     assert( pCsr->pgno==1 );
   }
+  if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1);
   rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0);
   return rc;
 }
diff --git a/test/dbpage.test b/test/dbpage.test
index 5a68d3caba..754bc17926 100644
--- a/test/dbpage.test
+++ b/test/dbpage.test
@@ -56,6 +56,14 @@ do_execsql_test 160 {
   INSERT INTO t2 VALUES(11,12,13);
   SELECT pgno, quote(substr(data,1,5)) FROM sqlite_dbpage('aux1');
 } {1 X'53514C6974' 2 X'0D00000001'}
+do_execsql_test 170 {
+  CREATE TABLE aux1.x3(x,y,z);
+  INSERT INTO x3(x,y,z) VALUES(1,'main',1),(2,'aux1',1);
+  SELECT pgno, schema, substr(data,1,6)
+    FROM sqlite_dbpage, x3
+   WHERE sqlite_dbpage.schema=x3.y AND sqlite_dbpage.pgno=x3.z
+   ORDER BY x3.x;
+} {1 main SQLite 1 aux1 SQLite}
 
 do_execsql_test 200 {
   CREATE TEMP TABLE saved_content(x);

From bde13e26949ea15d639724a0d8d301ead3ab8005 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 27 Oct 2017 20:53:16 +0000
Subject: [PATCH 224/270] Add the start of the "incremental_index_check"
 virtual table in ext/repair/checkindex.c. For incremental verification of
 index contents.

FossilOrigin-Name: d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7
---
 ext/repair/checkindex.c | 442 ++++++++++++++++++++++++++++++++++++++++
 manifest                |  17 +-
 manifest.uuid           |   2 +-
 test/checkindex.test    |  91 +++++++++
 4 files changed, 545 insertions(+), 7 deletions(-)
 create mode 100644 ext/repair/checkindex.c
 create mode 100644 test/checkindex.test

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
new file mode 100644
index 0000000000..a74d1d4b75
--- /dev/null
+++ b/ext/repair/checkindex.c
@@ -0,0 +1,442 @@
+/*
+** 2017 October 27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+#ifndef SQLITE_AMALGAMATION
+# include <string.h>
+# include <stdio.h>
+# include <stdlib.h>
+# include <assert.h>
+# define ALWAYS(X)  1
+# define NEVER(X)   0
+  typedef unsigned char u8;
+  typedef unsigned short u16;
+  typedef unsigned int u32;
+#define get4byte(x) (        \
+    ((u32)((x)[0])<<24) +    \
+    ((u32)((x)[1])<<16) +    \
+    ((u32)((x)[2])<<8) +     \
+    ((u32)((x)[3]))          \
+)
+#endif
+
+typedef struct CidxTable CidxTable;
+typedef struct CidxCursor CidxCursor;
+
+struct CidxTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;
+};
+
+struct CidxCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_stmt *pStmt;
+};
+
+static void *cidxMalloc(int *pRc, int n){
+  void *pRet = 0;
+  assert( n!=0 );
+  if( *pRc==SQLITE_OK ){
+    pRet = sqlite3_malloc(n);
+    if( pRet ){
+      memset(pRet, 0, n);
+    }else{
+      *pRc = SQLITE_NOMEM;
+    }
+  }
+  return pRet;
+}
+
+static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  assert( pCsr->base.pVtab->zErrMsg==0 );
+  pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+/*
+** Connect to then incremental_index_check virtual table.
+*/
+static int cidxConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  CidxTable *pRet;
+
+  rc = sqlite3_declare_vtab(db,
+      "CREATE TABLE xyz("
+      " errmsg TEXT, current_key TEXT,"
+      " index_name HIDDEN, after_key HIDDEN"
+      ")"
+  );
+  pRet = cidxMalloc(&rc, sizeof(CidxTable));
+  if( pRet ){
+    pRet->db = db;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pRet;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy an incremental_index_check virtual table.
+*/
+static int cidxDisconnect(sqlite3_vtab *pVtab){
+  CidxTable *pTab = (CidxTable*)pVtab;
+  sqlite3_free(pTab);
+  return SQLITE_OK;
+}
+
+/*
+** xBestIndex method.
+*/
+static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){
+  int iIdxName = -1;
+  int iAfterKey = -1;
+  int i;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
+    if( p->usable==0 ) continue;
+    if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+
+    if( p->iColumn==2 ){
+      iIdxName = i;
+    }
+    if( p->iColumn==3 ){
+      iAfterKey = i;
+    }
+  }
+
+  if( iIdxName<0 ){
+    pInfo->estimatedCost = 1000000000.0;
+  }else{
+    pInfo->aConstraintUsage[iIdxName].argvIndex = 1;
+    pInfo->aConstraintUsage[iIdxName].omit = 1;
+    if( iAfterKey<0 ){
+      pInfo->estimatedCost = 1000000.0;
+    }else{
+      pInfo->aConstraintUsage[iAfterKey].argvIndex = 2;
+      pInfo->aConstraintUsage[iAfterKey].omit = 1;
+      pInfo->estimatedCost = 1000.0;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  CidxCursor *pRet;
+  int rc = SQLITE_OK;
+
+  pRet = cidxMalloc(&rc, sizeof(CidxCursor));
+
+  *ppCursor = (sqlite3_vtab_cursor*)pRet;
+  return rc;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int cidxClose(sqlite3_vtab_cursor *pCursor){
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  pCsr->pStmt = 0;
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int cidxNext(sqlite3_vtab_cursor *pCursor){
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  int rc = sqlite3_step(pCsr->pStmt);
+  if( rc!=SQLITE_ROW ){
+    rc = sqlite3_finalize(pCsr->pStmt);
+    pCsr->pStmt = 0;
+    if( rc!=SQLITE_OK ){
+      sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db;
+      cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db));
+    }
+  }else{
+    rc = SQLITE_OK;
+  }
+  return rc;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int cidxEof(sqlite3_vtab_cursor *pCursor){
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  return pCsr->pStmt==0;
+}
+
+static sqlite3_stmt *cidxPrepare(
+  int *pRc, CidxCursor *pCsr, const char *zFmt, ...
+){
+  sqlite3_stmt *pRet = 0;
+  char *zSql;
+  va_list ap;                     /* ... printf arguments */
+  va_start(ap, zFmt);
+
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( *pRc==SQLITE_OK ){
+    if( zSql==0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db;
+      *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0);
+      if( *pRc!=SQLITE_OK ){
+        cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db));
+      }
+    }
+  }
+  sqlite3_free(zSql);
+  va_end(ap);
+
+  return pRet;
+}
+
+static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+char *cidxStrdup(int *pRc, const char *zStr){
+  char *zRet = 0;
+  if( *pRc==SQLITE_OK ){
+    int n = strlen(zStr);
+    zRet = cidxMalloc(pRc, n+1);
+    if( zRet ) memcpy(zRet, zStr, n+1);
+  }
+  return zRet;
+}
+
+static int cidxLookupIndex(
+  CidxCursor *pCsr,               /* Cursor object */
+  const char *zIdx,               /* Name of index to look up */
+  char **pzTab,                   /* OUT: Table name */
+  char **pzCurrentKey,            /* OUT: Expression for current_key */
+  char **pzOrderBy,               /* OUT: ORDER BY expression list */
+  char **pzSubWhere,              /* OUT: sub-query WHERE clause */
+  char **pzSubExpr                /* OUT: sub-query WHERE clause */
+){
+  int rc = SQLITE_OK;
+  char *zTab = 0;
+  char *zCurrentKey = 0;
+  char *zOrderBy = 0;
+  char *zSubWhere = 0;
+  char *zSubExpr = 0;
+
+  sqlite3_stmt *pFindTab = 0;
+  sqlite3_stmt *pGroup = 0;
+    
+  /* Find the table */
+  pFindTab = cidxPrepare(&rc, pCsr, 
+      "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'",
+      zIdx
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){
+    zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0));
+  }
+  cidxFinalize(&rc, pFindTab);
+  if( rc==SQLITE_OK && zTab==0 ){
+    rc = SQLITE_ERROR;
+  }
+
+  pGroup = cidxPrepare(&rc, pCsr,
+      "SELECT group_concat("
+      "  coalesce(name, 'rowid'), '|| '','' ||'"
+      ") AS zCurrentKey,"
+      "       group_concat("
+      "  coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END,"
+      "  ', '"
+      ") AS zOrderBy,"
+      "       group_concat("
+      "         CASE WHEN key==1 THEN NULL ELSE "
+      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
+      "         END,"
+      "  'AND '"
+      ") AS zSubWhere,"
+      "       group_concat("
+      "         CASE WHEN key==0 THEN NULL ELSE "
+      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
+      "         END,"
+      "  'AND '"
+      ") AS zSubExpr "
+      " FROM pragma_index_xinfo(%Q);"
+      , zIdx, zIdx, zIdx
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){
+    zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0));
+    zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1));
+    zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2));
+    zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3));
+  }
+  cidxFinalize(&rc, pGroup);
+  
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(zTab);
+    sqlite3_free(zCurrentKey);
+    sqlite3_free(zOrderBy);
+    sqlite3_free(zSubWhere);
+    sqlite3_free(zSubExpr);
+  }else{
+    *pzTab = zTab;
+    *pzCurrentKey = zCurrentKey;
+    *pzOrderBy = zOrderBy;
+    *pzSubWhere = zSubWhere;
+    *pzSubExpr = zSubExpr;
+  }
+
+  return rc;
+}
+
+/* 
+** Position a cursor back to the beginning.
+*/
+static int cidxFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  int rc = SQLITE_OK;
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  const char *zIdxName = 0;
+  const char *zAfterKey = 0;
+
+  if( argc>0 ){
+    zIdxName = (const char*)sqlite3_value_text(argv[0]);
+    if( argc>1 ){
+      zAfterKey = (const char*)sqlite3_value_text(argv[1]);
+    }
+  }
+
+  if( zIdxName ){
+    char *zTab = 0;
+    char *zCurrentKey = 0;
+    char *zOrderBy = 0;
+    char *zSubWhere = 0;
+    char *zSubExpr = 0;
+
+    rc = cidxLookupIndex(pCsr, zIdxName, 
+        &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr
+    );
+    pCsr->pStmt = cidxPrepare(&rc, pCsr, 
+        "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
+        zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
+    );
+
+    sqlite3_free(zTab);
+    sqlite3_free(zCurrentKey);
+    sqlite3_free(zOrderBy);
+    sqlite3_free(zSubWhere);
+    sqlite3_free(zSubExpr);
+  }
+
+  if( pCsr->pStmt ){
+    assert( rc==SQLITE_OK );
+    rc = cidxNext(pCursor);
+  }
+  return rc;
+}
+
+/* Return a column for the sqlite_btreeinfo table */
+static int cidxColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int iCol
+){
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  assert( iCol==0 || iCol==1 );
+  if( iCol==0 ){
+    const char *zVal = 0;
+    if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){
+      if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){
+        zVal = "row data mismatch";
+      }
+    }else{
+      zVal = "row missing";
+    }
+    sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC);
+  }else{
+    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1));
+  }
+  return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  *pRowid = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Register the virtual table modules with the database handle passed
+** as the only argument.
+*/
+static int ciInit(sqlite3 *db){
+  static sqlite3_module cidx_module = {
+    0,                            /* iVersion */
+    0,                            /* xCreate */
+    cidxConnect,                  /* xConnect */
+    cidxBestIndex,                /* xBestIndex */
+    cidxDisconnect,               /* xDisconnect */
+    0,                            /* xDestroy */
+    cidxOpen,                     /* xOpen - open a cursor */
+    cidxClose,                    /* xClose - close a cursor */
+    cidxFilter,                   /* xFilter - configure scan constraints */
+    cidxNext,                     /* xNext - advance a cursor */
+    cidxEof,                      /* xEof - check for end of scan */
+    cidxColumn,                   /* xColumn - read data */
+    cidxRowid,                    /* xRowid - read data */
+    0,                            /* xUpdate */
+    0,                            /* xBegin */
+    0,                            /* xSync */
+    0,                            /* xCommit */
+    0,                            /* xRollback */
+    0,                            /* xFindMethod */
+    0,                            /* xRename */
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+  };
+  return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0);
+}
+
+/*
+** Extension load function.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_checkindex_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi);
+  return ciInit(db);
+}
diff --git a/manifest b/manifest
index 2d13f91b8d..e0740cdd7b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32.
-D 2017-10-27T17:09:44.546
+C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents.
+D 2017-10-27T20:53:16.451
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,6 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
+F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -653,6 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
+F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1666,7 +1668,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f
-R b04098ed5c4b7451669ce8c4fddf24e2
-U mistachkin
-Z 04b0a33e7f8b0430861d645ce26e5f19
+P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
+R 38fb5f31777282a52c65b4e266007341
+T *branch * checkindex
+T *sym-checkindex *
+T -sym-trunk *
+U dan
+Z 12607f1e180da6aa96255bb6397017d4
diff --git a/manifest.uuid b/manifest.uuid
index 478c761165..9ee1c84b8f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
\ No newline at end of file
+d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
new file mode 100644
index 0000000000..841dca75a2
--- /dev/null
+++ b/test/checkindex.test
@@ -0,0 +1,91 @@
+# 2017-10-11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the checkindex extension.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix checkindex
+
+ifcapable !vtab||!compound {
+  finish_test
+  return
+}
+
+if {[file exists ../checkindex.so]==0} {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a);
+  INSERT INTO t1 VALUES('one', 2);
+  INSERT INTO t1 VALUES('two', 4);
+  INSERT INTO t1 VALUES('three', 6);
+  INSERT INTO t1 VALUES('four', 8);
+  INSERT INTO t1 VALUES('five', 10);
+}
+
+db enable_load_extension 1
+do_execsql_test 1.1 {
+  SELECT load_extension('../checkindex.so');
+} {{}}
+
+do_execsql_test 1.2 {
+  SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
+} {
+  1 five,5
+  1 four,4
+  1 one,1
+  1 three,3
+  1 two,2
+}
+
+
+do_test 1.3 {
+  set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }]
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
+  db eval {CREATE TABLE xt1(a, b)}
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
+
+  execsql {
+    UPDATE xt1 SET a='six' WHERE rowid=3;
+    DELETE FROM xt1 WHERE rowid = 5;
+  }
+
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+} {}
+
+do_execsql_test 1.4 {
+  SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
+} {
+  0 five,5
+  1 four,4
+  1 one,1
+  0 three,3
+  1 two,2
+}
+do_execsql_test 1.5 {
+  SELECT errmsg, current_key FROM incremental_index_check('i1');
+} {
+  {row missing} five,5
+  {} four,4
+  {} one,1
+  {row data mismatch} three,3
+  {} two,2
+}
+
+
+
+finish_test
+

From 6d6e76f75faaf018a03672794bc65805e6c1207e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 28 Oct 2017 12:20:09 +0000
Subject: [PATCH 225/270] Add test cases from OSSFuzz to prevent a regression
 in co-routine processing.

FossilOrigin-Name: 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 test/with1.test | 13 ++++++++++++-
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7abf007c96..723da88b0d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sPager\sref-count\sleak\sin\sthe\ssqlite_dbpage\svirtual\stable.
-D 2017-10-27T18:24:11.866
+C Add\stest\scases\sfrom\sOSSFuzz\sto\sprevent\sa\sregression\sin\sco-routine\nprocessing.
+D 2017-10-28T12:20:09.635
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -1557,7 +1557,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
-F test/with1.test 732e3ef398dcecb609839cd5ef0cb63beb2a9eff31420f3b745fc55b9e85b61e
+F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced
 F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87
 F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
-R f85ba3328211fce31f6fd8c8e66f1d27
+P c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2
+R 49302fb645a8bd5bf1702bc6c1b70644
 U drh
-Z a44af1749e0d20716f81debef1ff6db8
+Z 9899cde33cecabbc451b9857a10233e0
diff --git a/manifest.uuid b/manifest.uuid
index f8e9e5b80a..0d14844e0c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2
\ No newline at end of file
+689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2
\ No newline at end of file
diff --git a/test/with1.test b/test/with1.test
index 252a32bc69..92fbc58fcc 100644
--- a/test/with1.test
+++ b/test/with1.test
@@ -1003,6 +1003,17 @@ do_execsql_test 19.1 {
   SELECT * FROM t1;
 } {0 0 0 {SCAN SUBQUERY 1} 0 1 1 {SCAN SUBQUERY 1}}
 
-
+# 2017-10-28.
+# See check-in https://sqlite.org/src/info/0926df095faf72c2
+# Tried to optimize co-routine processing by changing a Copy opcode
+# into SCopy.  But OSSFuzz found two (similar) cases where that optimization
+# does not work.
+#
+do_execsql_test 20.1 {
+  WITH c(i)AS(VALUES(9)UNION SELECT~i FROM c)SELECT max(5)>i fROM c;
+} {0}
+do_execsql_test 20.2 {
+  WITH c(i)AS(VALUES(5)UNIoN SELECT 0)SELECT min(1)-i fROM c;
+} {1}
 
 finish_test

From 19465e7380f725f9bada9c338e55d7e95bf46ef3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 28 Oct 2017 20:31:25 +0000
Subject: [PATCH 226/270] Fix "after" parameter handling in the
 incremental_index_check code.

FossilOrigin-Name: c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
---
 ext/repair/checkindex.c | 238 ++++++++++++++++++++++++++++++++++++++--
 manifest                |  17 ++-
 manifest.uuid           |   2 +-
 test/checkindex.test    | 175 +++++++++++++++++++++++++----
 4 files changed, 389 insertions(+), 43 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index a74d1d4b75..f86cfdae7a 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -45,6 +45,13 @@ struct CidxCursor {
   sqlite3_stmt *pStmt;
 };
 
+typedef struct CidxColumn CidxColumn;
+struct CidxColumn {
+  char *zName;
+  char *zColl;
+  int bDesc;
+};
+
 static void *cidxMalloc(int *pRc, int n){
   void *pRet = 0;
   assert( n!=0 );
@@ -193,6 +200,23 @@ static int cidxEof(sqlite3_vtab_cursor *pCursor){
   return pCsr->pStmt==0;
 }
 
+static char *cidxMprintf(int *pRc, const char *zFmt, ...){
+  char *zRet = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zRet = sqlite3_vmprintf(zFmt, ap);
+  if( *pRc==SQLITE_OK ){
+    if( zRet==0 ){
+      *pRc = SQLITE_NOMEM;
+    }
+  }else{
+    sqlite3_free(zRet);
+    zRet = 0;
+  }
+  va_end(ap);
+  return zRet;
+}
+
 static sqlite3_stmt *cidxPrepare(
   int *pRc, CidxCursor *pCsr, const char *zFmt, ...
 ){
@@ -237,6 +261,8 @@ char *cidxStrdup(int *pRc, const char *zStr){
 static int cidxLookupIndex(
   CidxCursor *pCsr,               /* Cursor object */
   const char *zIdx,               /* Name of index to look up */
+  int *pnCol,                     /* OUT: Number of columns in index */
+  CidxColumn **paCol,             /* OUT: Columns */
   char **pzTab,                   /* OUT: Table name */
   char **pzCurrentKey,            /* OUT: Expression for current_key */
   char **pzOrderBy,               /* OUT: ORDER BY expression list */
@@ -249,6 +275,7 @@ static int cidxLookupIndex(
   char *zOrderBy = 0;
   char *zSubWhere = 0;
   char *zSubExpr = 0;
+  CidxColumn *aCol = 0;
 
   sqlite3_stmt *pFindTab = 0;
   sqlite3_stmt *pGroup = 0;
@@ -268,7 +295,7 @@ static int cidxLookupIndex(
 
   pGroup = cidxPrepare(&rc, pCsr,
       "SELECT group_concat("
-      "  coalesce(name, 'rowid'), '|| '','' ||'"
+      "  coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'"
       ") AS zCurrentKey,"
       "       group_concat("
       "  coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END,"
@@ -278,14 +305,15 @@ static int cidxLookupIndex(
       "         CASE WHEN key==1 THEN NULL ELSE "
       "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
       "         END,"
-      "  'AND '"
+      "  ' AND '"
       ") AS zSubWhere,"
       "       group_concat("
       "         CASE WHEN key==0 THEN NULL ELSE "
       "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
       "         END,"
-      "  'AND '"
-      ") AS zSubExpr "
+      "  ' AND '"
+      ") AS zSubExpr,"
+      "       count(*) AS nCol"
       " FROM pragma_index_xinfo(%Q);"
       , zIdx, zIdx, zIdx
   );
@@ -294,8 +322,49 @@ static int cidxLookupIndex(
     zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1));
     zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2));
     zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3));
+    *pnCol = sqlite3_column_int(pGroup, 4);
   }
   cidxFinalize(&rc, pGroup);
+
+  pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+  if( rc==SQLITE_OK ){
+    int nByte = 0;
+    int nCol = 0;
+    while( sqlite3_step(pGroup)==SQLITE_ROW ){
+      const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
+      const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
+      if( zName==0 ) zName = "rowid";
+      nCol++;
+      nByte += strlen(zName)+1 + strlen(zColl)+1;
+    }
+    rc = sqlite3_reset(pGroup);
+    aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte);
+
+    if( rc==SQLITE_OK ){
+      int iCol = 0;
+      char *z = (char*)&aCol[nCol];
+      while( sqlite3_step(pGroup)==SQLITE_ROW ){
+        int nName, nColl;
+        const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
+        const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
+        if( zName==0 ) zName = "rowid";
+
+        nName = strlen(zName);
+        nColl = strlen(zColl);
+        memcpy(z, zName, nName);
+        aCol[iCol].zName = z;
+        z += nName+1;
+
+        memcpy(z, zColl, nColl);
+        aCol[iCol].zColl = z;
+        z += nColl+1;
+
+        aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3);
+        iCol++;
+      }
+    }
+    cidxFinalize(&rc, pGroup);
+  }
   
   if( rc!=SQLITE_OK ){
     sqlite3_free(zTab);
@@ -303,17 +372,128 @@ static int cidxLookupIndex(
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
+    sqlite3_free(aCol);
   }else{
     *pzTab = zTab;
     *pzCurrentKey = zCurrentKey;
     *pzOrderBy = zOrderBy;
     *pzSubWhere = zSubWhere;
     *pzSubExpr = zSubExpr;
+    *paCol = aCol;
   }
 
   return rc;
 }
 
+static int cidxDecodeAfter(
+  CidxCursor *pCsr, 
+  int nCol, 
+  const char *zAfterKey, 
+  char ***pazAfter
+){
+  char **azAfter;
+  int rc = SQLITE_OK;
+  int nAfterKey = strlen(zAfterKey);
+
+  azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1);
+  if( rc==SQLITE_OK ){
+    int i;
+    char *zCopy = (char*)&azAfter[nCol];
+    char *p = zCopy;
+    memcpy(zCopy, zAfterKey, nAfterKey+1);
+    for(i=0; i<nCol; i++){
+      while( *p==' ' ) p++;
+
+      /* Check NULL values */
+      if( *p=='N' ){
+        if( memcmp(p, "NULL", 4) ) goto parse_error;
+        p += 4;
+      }
+
+      /* Check strings and blob literals */
+      else if( *p=='X' || *p=='\'' ){
+        azAfter[i] = p;
+        if( *p=='X' ) p++;
+        if( *p!='\'' ) goto parse_error;
+        p++;
+        while( 1 ){
+          if( *p=='\0' ) goto parse_error;
+          if( *p=='\'' ){
+            p++;
+            if( *p!='\'' ) break;
+          }
+          p++;
+        }
+      }
+
+      /* Check numbers */
+      else{
+        azAfter[i] = p;
+        while( (*p>='0' && *p<='9') 
+            || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E'
+        ){
+          p++;
+        }
+      }
+
+      while( *p==' ' ) p++;
+      if( *p!=(i==(nCol-1) ? '\0' : ',') ){
+        goto parse_error;
+      }
+      *p++ = '\0';
+    }
+  }
+
+  *pazAfter = azAfter;
+  return rc;
+
+ parse_error:
+  sqlite3_free(azAfter);
+  *pazAfter = 0;
+  cidxCursorError(pCsr, "%s", "error parsing after value");
+  return SQLITE_ERROR;
+}
+
+static char *cidxWhere(
+  int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull
+){
+  char *zRet = 0;
+  const char *zSep = "";
+  int i;
+
+  for(i=0; i<iGt; i++){
+    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s IS %s", zRet, 
+        zSep, aCol[i].zName, aCol[i].zColl, (azAfter[i] ? azAfter[i] : "NULL")
+    );
+    zSep = " AND ";
+  }
+
+  if( bLastIsNull ){
+    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zName);
+  }
+  else if( azAfter[iGt] ){
+    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s %s %s", zRet, 
+        zSep, aCol[iGt].zName, aCol[iGt].zColl, (aCol[iGt].bDesc ? "<" : ">"), 
+        azAfter[iGt]
+    );
+  }else{
+    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName);
+  }
+
+  return zRet;
+}
+
+static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){
+  int i;
+  char *zRet = 0;
+  const char *zSep = "";
+  for(i=0; i<nCol; i++){
+    zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, aCol[i].zName);
+    zSep = ",";
+  }
+  return zRet;
+}
+
 /* 
 ** Position a cursor back to the beginning.
 */
@@ -335,25 +515,65 @@ static int cidxFilter(
   }
 
   if( zIdxName ){
+    int nCol = 0;
     char *zTab = 0;
     char *zCurrentKey = 0;
     char *zOrderBy = 0;
     char *zSubWhere = 0;
     char *zSubExpr = 0;
+    char **azAfter = 0;
+    CidxColumn *aCol = 0;
 
     rc = cidxLookupIndex(pCsr, zIdxName, 
-        &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr
-    );
-    pCsr->pStmt = cidxPrepare(&rc, pCsr, 
-        "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
-        zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
+        &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr
     );
 
+    if( rc==SQLITE_OK && zAfterKey ){
+      rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter);
+    }
+
+    if( rc || zAfterKey==0 ){
+      pCsr->pStmt = cidxPrepare(&rc, pCsr, 
+          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
+          zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
+      );
+      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
+    }else{
+      char *zList = cidxColumnList(&rc, aCol, nCol);
+      const char *zSep = "";
+      char *zSql;
+      int i;
+
+      zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+          zSubExpr, zTab, zSubWhere, zCurrentKey
+      );
+      for(i=nCol-1; i>=0; i--){
+        int j;
+        if( aCol[i].bDesc && azAfter[i]==0 ) continue;
+        for(j=0; j<2; j++){
+          char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j);
+          zSql = cidxMprintf(&rc, 
+              "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
+              zSql, zSep, zList, zTab, zWhere, zOrderBy
+              );
+          zSep = " UNION ALL ";
+          if( aCol[i].bDesc==0 ) break;
+        }
+      }
+      zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName);
+      sqlite3_free(zList);
+
+      /* printf("SQL: %s\n", zSql); */
+      pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql);
+    }
+
     sqlite3_free(zTab);
     sqlite3_free(zCurrentKey);
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
+    sqlite3_free(aCol);
+    sqlite3_free(azAfter);
   }
 
   if( pCsr->pStmt ){
diff --git a/manifest b/manifest
index e0740cdd7b..8f56822a50 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents.
-D 2017-10-27T20:53:16.451
+C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode.
+D 2017-10-28T20:31:25.564
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1
+F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4
+F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,10 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de
-R 38fb5f31777282a52c65b4e266007341
-T *branch * checkindex
-T *sym-checkindex *
-T -sym-trunk *
+P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7
+R 31f10dd93d0cbe363e20ad50a747a726
 U dan
-Z 12607f1e180da6aa96255bb6397017d4
+Z 019388109f5e5cd896dce5eabbc0252a
diff --git a/manifest.uuid b/manifest.uuid
index 9ee1c84b8f..45cec907d3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7
\ No newline at end of file
+c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
index 841dca75a2..bafa42334c 100644
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -34,6 +34,8 @@ do_execsql_test 1.0 {
   INSERT INTO t1 VALUES('three', 6);
   INSERT INTO t1 VALUES('four', 8);
   INSERT INTO t1 VALUES('five', 10);
+
+  CREATE INDEX i2 ON t1(a DESC);
 }
 
 db enable_load_extension 1
@@ -41,18 +43,61 @@ do_execsql_test 1.1 {
   SELECT load_extension('../checkindex.so');
 } {{}}
 
+proc incr_index_check {idx nStep} {
+  set Q {
+    SELECT errmsg, current_key FROM incremental_index_check($idx, $after)
+    LIMIT $nStep
+  }
+
+  set res [list]
+  while {1} {
+    unset -nocomplain current_key
+    set res1 [db eval $Q]
+    if {[llength $res1]==0} break
+    set res [concat $res $res1]
+    set after [lindex $res end]
+  }
+
+  return $res
+}
+
+proc do_index_check_test {tn idx res} {
+  uplevel [list do_execsql_test $tn.1 "
+    SELECT errmsg, current_key FROM incremental_index_check('$idx');
+  " $res]
+
+  uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]]
+  #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
+  #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
+}
+
 do_execsql_test 1.2 {
   SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
 } {
-  1 five,5
-  1 four,4
-  1 one,1
-  1 three,3
-  1 two,2
+  1 'five',5
+  1 'four',4
+  1 'one',1
+  1 'three',3
+  1 'two',2
 }
 
+do_index_check_test 1.3 i1 {
+  {} 'five',5
+  {} 'four',4
+  {} 'one',1
+  {} 'three',3
+  {} 'two',2
+}
 
-do_test 1.3 {
+do_index_check_test 1.4 i2 {
+  {} 'two',2
+  {} 'three',3
+  {} 'one',1
+  {} 'four',4
+  {} 'five',5
+}
+
+do_test 1.5 {
   set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }]
   sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
   db eval {CREATE TABLE xt1(a, b)}
@@ -66,26 +111,110 @@ do_test 1.3 {
   sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
 } {}
 
-do_execsql_test 1.4 {
-  SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
-} {
-  0 five,5
-  1 four,4
-  1 one,1
-  0 three,3
-  1 two,2
-}
-do_execsql_test 1.5 {
-  SELECT errmsg, current_key FROM incremental_index_check('i1');
-} {
-  {row missing} five,5
-  {} four,4
-  {} one,1
-  {row data mismatch} three,3
-  {} two,2
+do_index_check_test 1.6 i1 {
+  {row missing} 'five',5
+  {} 'four',4
+  {} 'one',1
+  {row data mismatch} 'three',3
+  {} 'two',2
 }
 
+do_index_check_test 1.7 i2 {
+  {} 'two',2
+  {row data mismatch} 'three',3
+  {} 'one',1
+  {} 'four',4
+  {row missing} 'five',5
+}
 
+#--------------------------------------------------------------------------
+do_execsql_test 2.0 {
+
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d);
+
+  INSERT INTO t2 VALUES(1, NULL, 1, 1);
+  INSERT INTO t2 VALUES(2, 1, NULL, 1);
+  INSERT INTO t2 VALUES(3, 1, 1, NULL);
+
+  INSERT INTO t2 VALUES(4, 2, 2, 1);
+  INSERT INTO t2 VALUES(5, 2, 2, 2);
+  INSERT INTO t2 VALUES(6, 2, 2, 3);
+
+  INSERT INTO t2 VALUES(7, 2, 2, 1);
+  INSERT INTO t2 VALUES(8, 2, 2, 2);
+  INSERT INTO t2 VALUES(9, 2, 2, 3);
+
+  CREATE INDEX i3 ON t2(b, c, d);
+  CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC);
+  CREATE INDEX i5 ON t2(d, c DESC, b);
+}
+
+do_index_check_test 2.1 i3 {
+  {} NULL,1,1,1 
+  {} 1,NULL,1,2 
+  {} 1,1,NULL,3 
+  {} 2,2,1,4 
+  {} 2,2,1,7 
+  {} 2,2,2,5
+  {} 2,2,2,8 
+  {} 2,2,3,6 
+  {} 2,2,3,9
+}
+
+do_index_check_test 2.2 i4 {
+  {} 2,2,3,6 
+  {} 2,2,3,9
+  {} 2,2,2,5
+  {} 2,2,2,8 
+  {} 2,2,1,4 
+  {} 2,2,1,7 
+  {} 1,1,NULL,3 
+  {} 1,NULL,1,2 
+  {} NULL,1,1,1 
+}
+
+do_index_check_test 2.3 i5 {
+  {} NULL,1,1,3 
+  {} 1,2,2,4 
+  {} 1,2,2,7 
+  {} 1,1,NULL,1 
+  {} 1,NULL,1,2 
+  {} 2,2,2,5 
+  {} 2,2,2,8 
+  {} 3,2,2,6 
+  {} 3,2,2,9
+}
+
+do_execsql_test 3.0 {
+
+  CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID;
+  CREATE INDEX t3wxy ON t3(w, x, y);
+  CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC);
+
+  INSERT INTO t3 VALUES(NULL, NULL, NULL, 1);
+  INSERT INTO t3 VALUES(NULL, NULL, NULL, 2);
+  INSERT INTO t3 VALUES(NULL, NULL, NULL, 3);
+
+  INSERT INTO t3 VALUES('a', NULL, NULL, 4);
+  INSERT INTO t3 VALUES('a', NULL, NULL, 5);
+  INSERT INTO t3 VALUES('a', NULL, NULL, 6);
+
+  INSERT INTO t3 VALUES('a', 'b', NULL, 7);
+  INSERT INTO t3 VALUES('a', 'b', NULL, 8);
+  INSERT INTO t3 VALUES('a', 'b', NULL, 9);
+
+}
+
+do_index_check_test 3.1 t3wxy {
+  {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 
+  {} 'a',NULL,NULL,4  {} 'a',NULL,NULL,5  {} 'a',NULL,NULL,6 
+  {} 'a','b',NULL,7   {} 'a','b',NULL,8   {} 'a','b',NULL,9 
+}
+do_index_check_test 3.2 t3wxy2 {
+  {} 'a','b',NULL,7   {} 'a','b',NULL,8   {} 'a','b',NULL,9 
+  {} 'a',NULL,NULL,4  {} 'a',NULL,NULL,5  {} 'a',NULL,NULL,6 
+  {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 
+}
 
 finish_test
 

From fca23557fe1d3d246de864b3ebffd186e098b884 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 28 Oct 2017 20:51:54 +0000
Subject: [PATCH 227/270] Reactivate query flattening when the result set of
 the outer query has no function calls or subqueries.  This is a partial
 reversal of check-in [c9104b59].  Co-routines are still preferred if the
 outer query has a complex result set, but for simple results sets, query
 flattening is used.  Check-in [4464f40ccd7] is completely backed out due to
 this change.

FossilOrigin-Name: d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f
---
 manifest        | 17 +++++++++--------
 manifest.uuid   |  2 +-
 src/expr.c      |  1 +
 src/select.c    | 48 +++++++++++++++++++++++++++++-------------------
 src/sqliteInt.h |  9 ++++++---
 5 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index 723da88b0d..c2b9f80f14 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\scases\sfrom\sOSSFuzz\sto\sprevent\sa\sregression\sin\sco-routine\nprocessing.
-D 2017-10-28T12:20:09.635
+C Reactivate\squery\sflattening\swhen\sthe\sresult\sset\sof\sthe\souter\squery\shas\nno\sfunction\scalls\sor\ssubqueries.\s\sThis\sis\sa\spartial\sreversal\sof\s\ncheck-in\s[c9104b59].\s\sCo-routines\sare\sstill\spreferred\sif\sthe\souter\nquery\shas\sa\scomplex\sresult\sset,\sbut\sfor\ssimple\sresults\ssets,\squery\sflattening\nis\sused.\s\sCheck-in\s[4464f40ccd7]\sis\scompletely\sbacked\nout\sdue\sto\sthis\schange.
+D 2017-10-28T20:51:54.404
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -416,7 +416,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
-F src/expr.c 755caeafc43e3cd31e1d810795712641f6e19f7e7e9575faece4c77381fd8304
+F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
@@ -463,12 +463,12 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 0b4c1ac59221c91c9f01c326105c1749874704a77310735c0417f3c3026a4f11
+F src/select.c 36345c63153e3e3e8ba533c6d3be6201c317358e8c2b34c51400abdf192515ab
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9
+F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1666,7 +1666,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c063bb04da77d6847e4e254b6bb939c860e781382e34ddff8e0454d9db7552e2
-R 49302fb645a8bd5bf1702bc6c1b70644
+P 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2
+Q -4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
+R 645c7260930835d4324fc70970813d97
 U drh
-Z 9899cde33cecabbc451b9857a10233e0
+Z 4445aa956123d14dc2e74ce6385c6a77
diff --git a/manifest.uuid b/manifest.uuid
index 0d14844e0c..d5d218e0f2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2
\ No newline at end of file
+d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 31e9a93865..0a5dc913f6 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -952,6 +952,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){
     return 0;
   }
   pNew->x.pList = pList;
+  ExprSetProperty(pNew, EP_HasFunc);
   assert( !ExprHasProperty(pNew, EP_xIsSelect) );
   sqlite3ExprSetHeightAndFlags(pParse, pNew);
   return pNew;
diff --git a/src/select.c b/src/select.c
index 69ef06aeb5..f31dd850ab 100644
--- a/src/select.c
+++ b/src/select.c
@@ -3383,12 +3383,11 @@ static void substSelect(
 **  (19)  If the subquery uses LIMIT then the outer query may not
 **        have a WHERE clause.
 **
-**  (**)  Subsumed into (17d3).  Was: If the sub-query is a compound select,
-**        then it must not use an ORDER BY clause - Ticket #3773.  Because
-**        of (17d3), then only way to have a compound subquery is if it is
-**        the only term in the FROM clause of the outer query.  But if the
-**        only term in the FROM clause has an ORDER BY, then it will be
-**        implemented as a co-routine and the flattener will never be called.
+**  (20)  If the sub-query is a compound select, then it must not use
+**        an ORDER BY clause.  Ticket #3773.  We could relax this constraint
+**        somewhat by saying that the terms of the ORDER BY clause must
+**        appear as unmodified result columns in the outer query.  But we
+**        have other optimizations in mind to deal with that case.
 **
 **  (21)  If the subquery uses LIMIT then the outer query may not be
 **        DISTINCT.  (See ticket [752e1646fc]).
@@ -3522,6 +3521,9 @@ static int flattenSubquery(
   ** queries.
   */
   if( pSub->pPrior ){
+    if( pSub->pOrderBy ){
+      return 0;  /* Restriction (20) */
+    }
     if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
       return 0; /* (17d1), (17d2), or (17d3) */
     }
@@ -3556,15 +3558,6 @@ static int flattenSubquery(
   */
   assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 );
 
-  /* Ex-restriction (20):
-  ** A compound subquery must be the only term in the FROM clause of the
-  ** outer query by restriction (17d3).  But if that term also has an
-  ** ORDER BY clause, then the subquery will be implemented by co-routine
-  ** and so the flattener will never be invoked.  Hence, it is not possible
-  ** for the subquery to be a compound and have an ORDER BY clause.
-  */
-  assert( pSub->pPrior==0 || pSub->pOrderBy==0 );
-
   /***** If we reach this point, flattening is permitted. *****/
   SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n",
                    pSub->zSelName, pSub, iFrom));
@@ -4343,6 +4336,7 @@ static int selectExpander(Walker *pWalker, Select *p){
   sqlite3 *db = pParse->db;
   Expr *pE, *pRight, *pExpr;
   u16 selFlags = p->selFlags;
+  u32 elistFlags = 0;
 
   p->selFlags |= SF_Expanded;
   if( db->mallocFailed  ){
@@ -4455,6 +4449,7 @@ static int selectExpander(Walker *pWalker, Select *p){
     assert( pE->op!=TK_DOT || pE->pRight!=0 );
     assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
     if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
+    elistFlags |= pE->flags;
   }
   if( k<pEList->nExpr ){
     /*
@@ -4470,6 +4465,7 @@ static int selectExpander(Walker *pWalker, Select *p){
 
     for(k=0; k<pEList->nExpr; k++){
       pE = a[k].pExpr;
+      elistFlags |= pE->flags;
       pRight = pE->pRight;
       assert( pE->op!=TK_DOT || pRight!=0 );
       if( pE->op!=TK_ASTERISK
@@ -4599,9 +4595,14 @@ static int selectExpander(Walker *pWalker, Select *p){
     sqlite3ExprListDelete(db, pEList);
     p->pEList = pNew;
   }
-  if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
-    sqlite3ErrorMsg(pParse, "too many columns in result set");
-    return WRC_Abort;
+  if( p->pEList ){
+    if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
+      sqlite3ErrorMsg(pParse, "too many columns in result set");
+      return WRC_Abort;
+    }
+    if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){
+      p->selFlags |= SF_ComplexResult;
+    }
   }
   return WRC_Continue;
 }
@@ -5225,7 +5226,9 @@ int sqlite3Select(
     if( (pSub->selFlags & SF_Aggregate)!=0 ) continue;
     assert( pSub->pGroupBy==0 );
 
-    /* If the subquery contains an ORDER BY clause and if
+    /* If the outer query contains a "complex" result set (that is,
+    ** if the result set of the outer query uses functions or subqueries)
+    ** and if the subquery contains an ORDER BY clause and if
     ** it will be implemented as a co-routine, then do not flatten.  This
     ** restriction allows SQL constructs like this:
     **
@@ -5234,9 +5237,16 @@ int sqlite3Select(
     **
     ** The expensive_function() is only computed on the 10 rows that
     ** are output, rather than every row of the table.
+    **
+    ** The requirement that the outer query have a complex result set
+    ** means that flattening does occur on simpler SQL constraints without
+    ** the expensive_function() like:
+    **
+    **  SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
     */
     if( pSub->pOrderBy!=0
      && i==0
+     && (p->selFlags & SF_ComplexResult)!=0
      && (pTabList->nSrc==1
          || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0)
     ){
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0cc435d7b7..378b085021 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2402,7 +2402,7 @@ struct Expr {
 */
 #define EP_FromJoin  0x000001 /* Originates in ON/USING clause of outer join */
 #define EP_Agg       0x000002 /* Contains one or more aggregate functions */
-                  /* 0x000004 // available for use */
+#define EP_HasFunc   0x000004 /* Contains one or more functions of any kind */
                   /* 0x000008 // available for use */
 #define EP_Distinct  0x000010 /* Aggregate function with DISTINCT keyword */
 #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */
@@ -2426,9 +2426,10 @@ struct Expr {
 #define EP_Leaf      0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */
 
 /*
-** Combinations of two or more EP_* flags
+** The EP_Propagate mask is a set of properties that automatically propagate
+** upwards into parent nodes.
 */
-#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */
+#define EP_Propagate (EP_Collate|EP_Subquery|EP_HasFunc)
 
 /*
 ** These macros can be used to test, set, or clear bits in the
@@ -2708,6 +2709,7 @@ struct NameContext {
 #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
 #define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
 #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
+#define NC_Complex   0x2000  /* True if a function or subquery seen */
 
 /*
 ** An instance of the following structure contains all information
@@ -2778,6 +2780,7 @@ struct Select {
 #define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
 #define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
 #define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
+#define SF_ComplexResult  0x40000  /* Result set contains subquery or function */
 
 
 /*

From b68db3187b893b12c1c690711e9318c524818d35 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 28 Oct 2017 20:54:51 +0000
Subject: [PATCH 228/270] Increase the version number for the next release -
 which is still months away but there have been significant query planner
 enhancements since the previous release.

FossilOrigin-Name: 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09
---
 VERSION       |  2 +-
 configure     | 18 +++++++++---------
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 4 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/VERSION b/VERSION
index 6075c9a9ff..a7e7070f80 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.21.0
+3.22.0
diff --git a/configure b/configure
index 64b95e912c..9e7aa526a2 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.21.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.22.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.21.0'
-PACKAGE_STRING='sqlite 3.21.0'
+PACKAGE_VERSION='3.22.0'
+PACKAGE_STRING='sqlite 3.22.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1464,7 +1464,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.22.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1529,7 +1529,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.21.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.22.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1654,7 +1654,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.21.0
+sqlite configure 3.22.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2073,7 +2073,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.21.0, which was
+It was created by sqlite $as_me 3.22.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -12167,7 +12167,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.21.0, which was
+This file was extended by sqlite $as_me 3.22.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12233,7 +12233,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.21.0
+sqlite config.status 3.22.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/manifest b/manifest
index c2b9f80f14..f18ede0142 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Reactivate\squery\sflattening\swhen\sthe\sresult\sset\sof\sthe\souter\squery\shas\nno\sfunction\scalls\sor\ssubqueries.\s\sThis\sis\sa\spartial\sreversal\sof\s\ncheck-in\s[c9104b59].\s\sCo-routines\sare\sstill\spreferred\sif\sthe\souter\nquery\shas\sa\scomplex\sresult\sset,\sbut\sfor\ssimple\sresults\ssets,\squery\sflattening\nis\sused.\s\sCheck-in\s[4464f40ccd7]\sis\scompletely\sbacked\nout\sdue\sto\sthis\schange.
-D 2017-10-28T20:51:54.404
+C Increase\sthe\sversion\snumber\sfor\sthe\snext\srelease\s-\swhich\sis\sstill\smonths\naway\sbut\sthere\shave\sbeen\ssignificant\squery\splanner\senhancements\ssince\sthe\nprevious\srelease.
+D 2017-10-28T20:54:51.351
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
-F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
+F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -30,7 +30,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure e9dbb65b61c36bb9622225de254b768d4816749ff4cb4d71307bb067095aceec x
+F configure bdc49e9f0b0ced903ebdb2850362dd3391eeb88585e0429d12b94928d2873b6b x
 F configure.ac 369ebae6c04d9d2de5064e21d300f2f42f2fbf13235cabff9d1a54f2b2c4d05d
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
@@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 689743d8e3fa81e65dcb067bbf61bab09210b3b39586c865c00d9f1d6692daf2
-Q -4464f40ccd7c5553f4d44120ca6dac4e9445f08f083f7dcb3bd66b4413d818e0
-R 645c7260930835d4324fc70970813d97
+P d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f
+R d5f45aed899d7f014af4553e25cf640f
 U drh
-Z 4445aa956123d14dc2e74ce6385c6a77
+Z 66b45fa3f9b3a864ae6a46fe2fdaa343
diff --git a/manifest.uuid b/manifest.uuid
index d5d218e0f2..0576f4ab05 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f
\ No newline at end of file
+457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09
\ No newline at end of file

From 6fc66179092c843b8f352579709417f6f5e5190d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Oct 2017 08:04:38 +0000
Subject: [PATCH 229/270] Fix an issue in incremental_index_check with indexes
 that use non-default collation sequences.

FossilOrigin-Name: 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
---
 ext/repair/checkindex.c |  5 +++--
 manifest                | 14 ++++++-------
 manifest.uuid           |  2 +-
 test/checkindex.test    | 44 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index f86cfdae7a..b70b57122c 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -298,7 +298,8 @@ static int cidxLookupIndex(
       "  coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'"
       ") AS zCurrentKey,"
       "       group_concat("
-      "  coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END,"
+      "  coalesce(name, 'rowid') || ' COLLATE ' || coll "
+      "  || CASE WHEN desc THEN ' DESC' ELSE '' END,"
       "  ', '"
       ") AS zOrderBy,"
       "       group_concat("
@@ -313,7 +314,7 @@ static int cidxLookupIndex(
       "         END,"
       "  ' AND '"
       ") AS zSubExpr,"
-      "       count(*) AS nCol"
+      "      count(*) AS nCol"
       " FROM pragma_index_xinfo(%Q);"
       , zIdx, zIdx, zIdx
   );
diff --git a/manifest b/manifest
index 8f56822a50..1aefe38098 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode.
-D 2017-10-28T20:31:25.564
+C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences.
+D 2017-10-30T08:04:38.448
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4
+F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966
+F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7
-R 31f10dd93d0cbe363e20ad50a747a726
+P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
+R 35b9a81e4b8b6b857354a831ff9c29c1
 U dan
-Z 019388109f5e5cd896dce5eabbc0252a
+Z cd0262c8d2e550da2f2d47bdfb3d2467
diff --git a/manifest.uuid b/manifest.uuid
index 45cec907d3..7ea946e219 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
\ No newline at end of file
+3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
index bafa42334c..706171ff19 100644
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -185,6 +185,7 @@ do_index_check_test 2.3 i5 {
   {} 3,2,2,9
 }
 
+#--------------------------------------------------------------------------
 do_execsql_test 3.0 {
 
   CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID;
@@ -216,5 +217,48 @@ do_index_check_test 3.2 t3wxy2 {
   {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 
 }
 
+#--------------------------------------------------------------------------
+# Test with an index that uses non-default collation sequences.
+#
+do_execsql_test 4.0 {
+  CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT);
+  INSERT INTO t4 VALUES(1, 'aaa', 'bbb');
+  INSERT INTO t4 VALUES(2, 'AAA', 'CCC');
+  INSERT INTO t4 VALUES(3, 'aab', 'ddd');
+  INSERT INTO t4 VALUES(4, 'AAB', 'EEE');
+
+  CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase);
+}
+
+do_index_check_test 4.1 t4cc {
+  {} 'aaa','bbb',1 
+  {} 'AAA','CCC',2 
+  {} 'aab','ddd',3 
+  {} 'AAB','EEE',4
+}
+
+do_test 4.2 {
+  set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }]
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
+  db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)}
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
+
+  execsql {
+    UPDATE xt4 SET c1='hello' WHERE rowid=2;
+    DELETE FROM xt4 WHERE rowid = 3;
+  }
+
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+} {}
+
+do_index_check_test 4.3 t4cc {
+  {} 'aaa','bbb',1 
+  {row data mismatch} 'AAA','CCC',2 
+  {row missing} 'aab','ddd',3 
+  {} 'AAB','EEE',4
+}
+
+
+
 finish_test
 

From 75650d4f9453db917f8198a8d276ed96590ffb8a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Oct 2017 17:05:18 +0000
Subject: [PATCH 230/270] In checkindex.c, use C code instead of
 SQL/group_concat() to compose various SQL clauses. This is to make it easier
 to support indexes on expressions.

FossilOrigin-Name: 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
---
 ext/repair/checkindex.c | 255 +++++++++++++++++++++-------------------
 manifest                |  14 +--
 manifest.uuid           |   2 +-
 test/checkindex.test    |   5 +-
 4 files changed, 144 insertions(+), 132 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index b70b57122c..0393358567 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -47,9 +47,15 @@ struct CidxCursor {
 
 typedef struct CidxColumn CidxColumn;
 struct CidxColumn {
-  char *zName;
-  char *zColl;
-  int bDesc;
+  char *zExpr;                    /* Text for indexed expression */
+  int bDesc;                      /* True for DESC columns, otherwise false */
+  int bKey;                       /* Part of index, not PK */
+};
+
+typedef struct CidxIndex CidxIndex;
+struct CidxIndex {
+  int nCol;                       /* Elements in aCol[] array */
+  CidxColumn aCol[1];             /* Array of indexed columns */
 };
 
 static void *cidxMalloc(int *pRc, int n){
@@ -258,29 +264,30 @@ char *cidxStrdup(int *pRc, const char *zStr){
   return zRet;
 }
 
+static void cidxFreeIndex(CidxIndex *pIdx){
+  if( pIdx ){
+    int i;
+    for(i=0; i<pIdx->nCol; i++){
+      sqlite3_free(pIdx->aCol[i].zExpr);
+    }
+    sqlite3_free(pIdx);
+  }
+}
+
 static int cidxLookupIndex(
   CidxCursor *pCsr,               /* Cursor object */
   const char *zIdx,               /* Name of index to look up */
-  int *pnCol,                     /* OUT: Number of columns in index */
-  CidxColumn **paCol,             /* OUT: Columns */
-  char **pzTab,                   /* OUT: Table name */
-  char **pzCurrentKey,            /* OUT: Expression for current_key */
-  char **pzOrderBy,               /* OUT: ORDER BY expression list */
-  char **pzSubWhere,              /* OUT: sub-query WHERE clause */
-  char **pzSubExpr                /* OUT: sub-query WHERE clause */
+  CidxIndex **ppIdx,              /* OUT: Description of columns */
+  char **pzTab                    /* OUT: Table name */
 ){
   int rc = SQLITE_OK;
   char *zTab = 0;
-  char *zCurrentKey = 0;
-  char *zOrderBy = 0;
-  char *zSubWhere = 0;
-  char *zSubExpr = 0;
-  CidxColumn *aCol = 0;
+  CidxIndex *pIdx = 0;
 
   sqlite3_stmt *pFindTab = 0;
-  sqlite3_stmt *pGroup = 0;
+  sqlite3_stmt *pInfo = 0;
     
-  /* Find the table */
+  /* Find the table for this index. */
   pFindTab = cidxPrepare(&rc, pCsr, 
       "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'",
       zIdx
@@ -293,94 +300,35 @@ static int cidxLookupIndex(
     rc = SQLITE_ERROR;
   }
 
-  pGroup = cidxPrepare(&rc, pCsr,
-      "SELECT group_concat("
-      "  coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'"
-      ") AS zCurrentKey,"
-      "       group_concat("
-      "  coalesce(name, 'rowid') || ' COLLATE ' || coll "
-      "  || CASE WHEN desc THEN ' DESC' ELSE '' END,"
-      "  ', '"
-      ") AS zOrderBy,"
-      "       group_concat("
-      "         CASE WHEN key==1 THEN NULL ELSE "
-      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
-      "         END,"
-      "  ' AND '"
-      ") AS zSubWhere,"
-      "       group_concat("
-      "         CASE WHEN key==0 THEN NULL ELSE "
-      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
-      "         END,"
-      "  ' AND '"
-      ") AS zSubExpr,"
-      "      count(*) AS nCol"
-      " FROM pragma_index_xinfo(%Q);"
-      , zIdx, zIdx, zIdx
-  );
-  if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){
-    zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0));
-    zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1));
-    zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2));
-    zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3));
-    *pnCol = sqlite3_column_int(pGroup, 4);
-  }
-  cidxFinalize(&rc, pGroup);
-
-  pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+  pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
   if( rc==SQLITE_OK ){
-    int nByte = 0;
-    int nCol = 0;
-    while( sqlite3_step(pGroup)==SQLITE_ROW ){
-      const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
-      const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
+    int nAlloc = 0;
+    int iCol = 0;
+
+    while( sqlite3_step(pInfo)==SQLITE_ROW ){
+      const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
+      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+      CidxColumn *p;
       if( zName==0 ) zName = "rowid";
-      nCol++;
-      nByte += strlen(zName)+1 + strlen(zColl)+1;
-    }
-    rc = sqlite3_reset(pGroup);
-    aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte);
-
-    if( rc==SQLITE_OK ){
-      int iCol = 0;
-      char *z = (char*)&aCol[nCol];
-      while( sqlite3_step(pGroup)==SQLITE_ROW ){
-        int nName, nColl;
-        const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
-        const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
-        if( zName==0 ) zName = "rowid";
-
-        nName = strlen(zName);
-        nColl = strlen(zColl);
-        memcpy(z, zName, nName);
-        aCol[iCol].zName = z;
-        z += nName+1;
-
-        memcpy(z, zColl, nColl);
-        aCol[iCol].zColl = z;
-        z += nColl+1;
-
-        aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3);
-        iCol++;
+      if( iCol==nAlloc ){
+        int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
+        pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
       }
+      p = &pIdx->aCol[iCol++];
+      p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
+      p->bDesc = sqlite3_column_int(pInfo, 3);
+      p->bKey = sqlite3_column_int(pInfo, 5);
+      pIdx->nCol = iCol;
     }
-    cidxFinalize(&rc, pGroup);
+    cidxFinalize(&rc, pInfo);
   }
   
   if( rc!=SQLITE_OK ){
     sqlite3_free(zTab);
-    sqlite3_free(zCurrentKey);
-    sqlite3_free(zOrderBy);
-    sqlite3_free(zSubWhere);
-    sqlite3_free(zSubExpr);
-    sqlite3_free(aCol);
+    cidxFreeIndex(pIdx);
   }else{
     *pzTab = zTab;
-    *pzCurrentKey = zCurrentKey;
-    *pzOrderBy = zOrderBy;
-    *pzSubWhere = zSubWhere;
-    *pzSubExpr = zSubExpr;
-    *paCol = aCol;
+    *ppIdx = pIdx;
   }
 
   return rc;
@@ -463,35 +411,91 @@ static char *cidxWhere(
   int i;
 
   for(i=0; i<iGt; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s IS %s", zRet, 
-        zSep, aCol[i].zName, aCol[i].zColl, (azAfter[i] ? azAfter[i] : "NULL")
+    zRet = cidxMprintf(pRc, "%z%s%s IS %s", zRet, 
+        zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL")
     );
     zSep = " AND ";
   }
 
   if( bLastIsNull ){
-    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zName);
+    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zExpr);
   }
   else if( azAfter[iGt] ){
-    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s %s %s", zRet, 
-        zSep, aCol[iGt].zName, aCol[iGt].zColl, (aCol[iGt].bDesc ? "<" : ">"), 
+    zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, 
+        zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), 
         azAfter[iGt]
     );
   }else{
-    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName);
+    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr);
   }
 
   return zRet;
 }
 
-static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){
-  int i;
+#define CIDX_CLIST_ALL         0
+#define CIDX_CLIST_ORDERBY     1
+#define CIDX_CLIST_CURRENT_KEY 2
+#define CIDX_CLIST_SUBWHERE    3
+#define CIDX_CLIST_SUBEXPR     4
+
+/*
+** This function returns various strings based on the contents of the
+** CidxIndex structure and the eType parameter.
+*/
+static char *cidxColumnList(
+  int *pRc,                       /* IN/OUT: Error code */
+  const char *zIdx,
+  CidxIndex *pIdx,                /* Indexed columns */
+  int eType                       /* True to include ASC/DESC */
+){
   char *zRet = 0;
-  const char *zSep = "";
-  for(i=0; i<nCol; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, aCol[i].zName);
-    zSep = ",";
+  if( *pRc==SQLITE_OK ){
+    const char *aDir[2] = {" ASC", " DESC"};
+    int i;
+    const char *zSep = "";
+
+    for(i=0; i<pIdx->nCol; i++){
+      CidxColumn *p = &pIdx->aCol[i];
+      assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 );
+      switch( eType ){
+
+        case CIDX_CLIST_ORDERBY:
+          zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]);
+          zSep = ",";
+          break;
+
+        case CIDX_CLIST_CURRENT_KEY:
+          zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr);
+          zSep = "||','||";
+          break;
+
+        case CIDX_CLIST_SUBWHERE:
+          if( p->bKey==0 ){
+            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
+                zSep, p->zExpr, zIdx, p->zExpr
+            );
+            zSep = " AND ";
+          }
+          break;
+
+        case CIDX_CLIST_SUBEXPR:
+          if( p->bKey==1 ){
+            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
+                zSep, p->zExpr, zIdx, p->zExpr
+            );
+            zSep = " AND ";
+          }
+          break;
+
+        default:
+          assert( eType==CIDX_CLIST_ALL );
+          zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr);
+          zSep = ",";
+          break;
+      }
+    }
   }
+
   return zRet;
 }
 
@@ -516,21 +520,26 @@ static int cidxFilter(
   }
 
   if( zIdxName ){
-    int nCol = 0;
     char *zTab = 0;
     char *zCurrentKey = 0;
     char *zOrderBy = 0;
     char *zSubWhere = 0;
     char *zSubExpr = 0;
-    char **azAfter = 0;
-    CidxColumn *aCol = 0;
+    char *zSrcList = 0;
 
-    rc = cidxLookupIndex(pCsr, zIdxName, 
-        &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr
-    );
+    char **azAfter = 0;
+    CidxIndex *pIdx = 0;
+
+    rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab);
+
+    zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY);
+    zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
+    zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
+    zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
+    /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */
 
     if( rc==SQLITE_OK && zAfterKey ){
-      rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter);
+      rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
     }
 
     if( rc || zAfterKey==0 ){
@@ -538,9 +547,9 @@ static int cidxFilter(
           "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
           zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
       );
-      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
+      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt));  */
     }else{
-      char *zList = cidxColumnList(&rc, aCol, nCol);
+      char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0);
       const char *zSep = "";
       char *zSql;
       int i;
@@ -548,17 +557,17 @@ static int cidxFilter(
       zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
           zSubExpr, zTab, zSubWhere, zCurrentKey
       );
-      for(i=nCol-1; i>=0; i--){
+      for(i=pIdx->nCol-1; i>=0; i--){
         int j;
-        if( aCol[i].bDesc && azAfter[i]==0 ) continue;
+        if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
         for(j=0; j<2; j++){
-          char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j);
+          char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
           zSql = cidxMprintf(&rc, 
               "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
               zSql, zSep, zList, zTab, zWhere, zOrderBy
               );
           zSep = " UNION ALL ";
-          if( aCol[i].bDesc==0 ) break;
+          if( pIdx->aCol[i].bDesc==0 ) break;
         }
       }
       zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName);
@@ -573,7 +582,7 @@ static int cidxFilter(
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
-    sqlite3_free(aCol);
+    cidxFreeIndex(pIdx);
     sqlite3_free(azAfter);
   }
 
@@ -584,7 +593,9 @@ static int cidxFilter(
   return rc;
 }
 
-/* Return a column for the sqlite_btreeinfo table */
+/* 
+** Return a column value.
+*/
 static int cidxColumn(
   sqlite3_vtab_cursor *pCursor, 
   sqlite3_context *ctx, 
diff --git a/manifest b/manifest
index 1aefe38098..48fe0a91a2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences.
-D 2017-10-30T08:04:38.448
+C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions.
+D 2017-10-30T17:05:18.290
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0
+F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2
+F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
-R 35b9a81e4b8b6b857354a831ff9c29c1
+P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
+R 45b731228881e6db68d5d1d151d2789f
 U dan
-Z cd0262c8d2e550da2f2d47bdfb3d2467
+Z 8f0b667700c59bebe92a45558d3d25e5
diff --git a/manifest.uuid b/manifest.uuid
index 7ea946e219..5d8e25d480 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
\ No newline at end of file
+940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
index 706171ff19..6648187559 100644
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -67,10 +67,11 @@ proc do_index_check_test {tn idx res} {
   " $res]
 
   uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]]
-  #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
-  #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
+  uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
+  uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
 }
 
+
 do_execsql_test 1.2 {
   SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
 } {

From eba21f9eb246db1e4e723ef9665c5bc5b6ec7749 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 30 Oct 2017 18:49:11 +0000
Subject: [PATCH 231/270] Improve the performance of the LIKE operator by using
 strcspn() to aid wildcard matching.

FossilOrigin-Name: 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/func.c    | 16 ++++++++++------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index f18ede0142..e88bdbcfad 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Increase\sthe\sversion\snumber\sfor\sthe\snext\srelease\s-\swhich\sis\sstill\smonths\naway\sbut\sthere\shave\sbeen\ssignificant\squery\splanner\senhancements\ssince\sthe\nprevious\srelease.
-D 2017-10-28T20:54:51.351
+C Improve\sthe\sperformance\sof\sthe\sLIKE\soperator\sby\susing\sstrcspn()\sto\said\nwildcard\smatching.
+D 2017-10-30T18:49:11.595
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -419,7 +419,7 @@ F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
-F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6
+F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d17ef7d153058f7332b3fec421ade42c67e26b06f36fc1629e6799537a5afc5f
-R d5f45aed899d7f014af4553e25cf640f
+P 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09
+R b136c0336cd9a2670f5306d89a5f1d34
 U drh
-Z 66b45fa3f9b3a864ae6a46fe2fdaa343
+Z 1ff3723c7287a54f5348d686239a8395
diff --git a/manifest.uuid b/manifest.uuid
index 0576f4ab05..e6859f5555 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09
\ No newline at end of file
+37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index b46578a407..7528fa8b4d 100644
--- a/src/func.c
+++ b/src/func.c
@@ -698,16 +698,20 @@ static int patternCompare(
       ** c or cx.
       */
       if( c<=0x80 ){
-        u32 cx;
+        char zStop[3];
         int bMatch;
         if( noCase ){
-          cx = sqlite3Toupper(c);
-          c = sqlite3Tolower(c);
+          zStop[0] = sqlite3Toupper(c);
+          zStop[1] = sqlite3Tolower(c);
+          zStop[2] = 0;
         }else{
-          cx = c;
+          zStop[0] = c;
+          zStop[1] = 0;
         }
-        while( (c2 = *(zString++))!=0 ){
-          if( c2!=c && c2!=cx ) continue;
+        while(1){
+          zString += strcspn((const char*)zString, zStop);
+          if( zString[0]==0 ) break;
+          zString++;
           bMatch = patternCompare(zPattern,zString,pInfo,matchOther);
           if( bMatch!=SQLITE_NOMATCH ) return bMatch;
         }

From e7a579d27d12ce936d5a536dffd7453bd0503235 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Oct 2017 19:38:41 +0000
Subject: [PATCH 232/270] Add support for indexes on expressions to
 incremental_index_check.

FossilOrigin-Name: 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
---
 ext/repair/checkindex.c | 215 +++++++++++++++++++++++++++++++---------
 manifest                |  14 +--
 manifest.uuid           |   2 +-
 test/checkindex.test    |  58 +++++++++++
 4 files changed, 232 insertions(+), 57 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index 0393358567..fc91b74244 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -274,6 +274,110 @@ static void cidxFreeIndex(CidxIndex *pIdx){
   }
 }
 
+#define CIDX_PARSE_EOF   0
+#define CIDX_PARSE_COMMA 1      /*  "," */
+#define CIDX_PARSE_OPEN  2      /*  "(" */
+#define CIDX_PARSE_CLOSE 3      /*  ")" */
+
+static int cidxFindNext(const char *zIn, const char **pzOut){
+  const char *z = zIn;
+
+  while( 1 ){
+    *pzOut = z;
+    switch( *z ){
+      case '\0':
+        return CIDX_PARSE_EOF;
+      case '(':
+        return CIDX_PARSE_OPEN;
+      case ')':
+        return CIDX_PARSE_CLOSE;
+      case ',':
+        return CIDX_PARSE_COMMA;
+
+      case '"': 
+      case '\'': 
+      case '`': {
+        char q = *z;
+        z++;
+        while( *z ){
+          if( *z==q ){
+            z++;
+            if( *z!=q ) break;
+          }
+          z++;
+        }
+        break;
+      }
+
+      case '[':
+        while( *z++!=']' );
+        break;
+
+      default:
+        z++;
+    }
+  }
+
+  assert( 0 );
+  return -1;
+}
+
+static int cidx_isspace(char c){
+  return c==' ' || c=='\t' || c=='\r' || c=='\n';
+}
+
+static int cidx_isident(char c){
+  return c<0 
+    || (c>='0' && c<='9') || (c>='a' && c<='z') 
+    || (c>='A' && c<='Z') || c=='_';
+}
+
+static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){
+  const char *z = zSql;
+  const char *z1;
+  int e;
+  int rc = SQLITE_OK;
+  int nParen = 1;
+  CidxColumn *pCol = pIdx->aCol;
+
+  e = cidxFindNext(z, &z);
+  if( e!=CIDX_PARSE_OPEN ) goto parse_error;
+  z1 = z+1;
+  z++;
+  while( nParen>0 ){
+    e = cidxFindNext(z, &z);
+    if( e==CIDX_PARSE_EOF ) goto parse_error;
+    if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){
+      const char *z2 = z;
+      if( pCol->zExpr ) goto parse_error;
+
+      while( cidx_isspace(z[-1]) ) z--;
+      if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){
+        z -= 3;
+        while( cidx_isspace(z[-1]) ) z--;
+      }else
+      if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){
+        z -= 4;
+        while( cidx_isspace(z[-1]) ) z--;
+      }
+
+      while( cidx_isspace(z1[0]) ) z1++;
+      pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1);
+      pCol++;
+      z = z1 = z2+1;
+    }
+    if( e==CIDX_PARSE_OPEN ) nParen++;
+    if( e==CIDX_PARSE_CLOSE ) nParen--;
+    z++;
+  }
+
+  return rc;
+
+ parse_error:
+  cidxCursorError(pCsr, "Parse error in: %s", zSql);
+  return SQLITE_ERROR;
+}
+
 static int cidxLookupIndex(
   CidxCursor *pCsr,               /* Cursor object */
   const char *zIdx,               /* Name of index to look up */
@@ -289,39 +393,50 @@ static int cidxLookupIndex(
     
   /* Find the table for this index. */
   pFindTab = cidxPrepare(&rc, pCsr, 
-      "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'",
+      "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'",
       zIdx
   );
   if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){
+    const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1);
     zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0));
+
+    pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+    if( rc==SQLITE_OK ){
+      int nAlloc = 0;
+      int iCol = 0;
+
+      while( sqlite3_step(pInfo)==SQLITE_ROW ){
+        const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
+        const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+        CidxColumn *p;
+        if( zName==0 ) zName = "rowid";
+        if( iCol==nAlloc ){
+          int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
+          pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
+          nAlloc += 8;
+        }
+        p = &pIdx->aCol[iCol++];
+        p->bDesc = sqlite3_column_int(pInfo, 3);
+        p->bKey = sqlite3_column_int(pInfo, 5);
+        if( zSql==0 || p->bKey==0 ){
+          p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
+        }else{
+          p->zExpr = 0;
+        }
+        pIdx->nCol = iCol;
+      }
+      cidxFinalize(&rc, pInfo);
+    }
+
+    if( rc==SQLITE_OK && zSql ){
+      rc = cidxParseSQL(pCsr, pIdx, zSql);
+    }
   }
+
   cidxFinalize(&rc, pFindTab);
   if( rc==SQLITE_OK && zTab==0 ){
     rc = SQLITE_ERROR;
   }
-
-  pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
-  if( rc==SQLITE_OK ){
-    int nAlloc = 0;
-    int iCol = 0;
-
-    while( sqlite3_step(pInfo)==SQLITE_ROW ){
-      const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
-      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
-      CidxColumn *p;
-      if( zName==0 ) zName = "rowid";
-      if( iCol==nAlloc ){
-        int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
-        pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
-      }
-      p = &pIdx->aCol[iCol++];
-      p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
-      p->bDesc = sqlite3_column_int(pInfo, 3);
-      p->bKey = sqlite3_column_int(pInfo, 5);
-      pIdx->nCol = iCol;
-    }
-    cidxFinalize(&rc, pInfo);
-  }
   
   if( rc!=SQLITE_OK ){
     sqlite3_free(zTab);
@@ -411,22 +526,22 @@ static char *cidxWhere(
   int i;
 
   for(i=0; i<iGt; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s IS %s", zRet, 
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS %s", zRet, 
         zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL")
     );
     zSep = " AND ";
   }
 
   if( bLastIsNull ){
-    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zExpr);
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS NULL", zRet, zSep, aCol[iGt].zExpr);
   }
   else if( azAfter[iGt] ){
-    zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, 
+    zRet = cidxMprintf(pRc, "%z%s(%s) %s %s", zRet, 
         zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), 
         azAfter[iGt]
     );
   }else{
-    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr);
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr);
   }
 
   return zRet;
@@ -450,7 +565,7 @@ static char *cidxColumnList(
 ){
   char *zRet = 0;
   if( *pRc==SQLITE_OK ){
-    const char *aDir[2] = {" ASC", " DESC"};
+    const char *aDir[2] = {"", " DESC"};
     int i;
     const char *zSep = "";
 
@@ -460,19 +575,19 @@ static char *cidxColumnList(
       switch( eType ){
 
         case CIDX_CLIST_ORDERBY:
-          zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]);
+          zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]);
           zSep = ",";
           break;
 
         case CIDX_CLIST_CURRENT_KEY:
-          zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr);
+          zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i);
           zSep = "||','||";
           break;
 
         case CIDX_CLIST_SUBWHERE:
           if( p->bKey==0 ){
-            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
-                zSep, p->zExpr, zIdx, p->zExpr
+            zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, 
+                zSep, p->zExpr, i
             );
             zSep = " AND ";
           }
@@ -480,8 +595,8 @@ static char *cidxColumnList(
 
         case CIDX_CLIST_SUBEXPR:
           if( p->bKey==1 ){
-            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
-                zSep, p->zExpr, zIdx, p->zExpr
+            zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, 
+                zSep, p->zExpr, i
             );
             zSep = " AND ";
           }
@@ -489,8 +604,8 @@ static char *cidxColumnList(
 
         default:
           assert( eType==CIDX_CLIST_ALL );
-          zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr);
-          zSep = ",";
+          zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i);
+          zSep = ", ";
           break;
       }
     }
@@ -536,7 +651,7 @@ static int cidxFilter(
     zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
     zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
     zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
-    /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */
+    zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL);
 
     if( rc==SQLITE_OK && zAfterKey ){
       rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
@@ -544,17 +659,19 @@ static int cidxFilter(
 
     if( rc || zAfterKey==0 ){
       pCsr->pStmt = cidxPrepare(&rc, pCsr, 
-          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
-          zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
+          "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
+          "FROM (SELECT %s FROM %Q ORDER BY %s) AS i",
+          zSubExpr, zTab, zSubWhere, zCurrentKey, 
+          zSrcList, zTab, zOrderBy
       );
-      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt));  */
+      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
     }else{
-      char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0);
       const char *zSep = "";
       char *zSql;
       int i;
 
-      zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+      zSql = cidxMprintf(&rc, 
+          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
           zSubExpr, zTab, zSubWhere, zCurrentKey
       );
       for(i=pIdx->nCol-1; i>=0; i--){
@@ -562,16 +679,15 @@ static int cidxFilter(
         if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
         for(j=0; j<2; j++){
           char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
-          zSql = cidxMprintf(&rc, 
-              "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
-              zSql, zSep, zList, zTab, zWhere, zOrderBy
-              );
+          zSql = cidxMprintf(&rc, "%z"
+              "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
+              zSql, zSep, zSrcList, zTab, zWhere, zOrderBy
+          );
           zSep = " UNION ALL ";
           if( pIdx->aCol[i].bDesc==0 ) break;
         }
       }
-      zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName);
-      sqlite3_free(zList);
+      zSql = cidxMprintf(&rc, "%z) AS i", zSql);
 
       /* printf("SQL: %s\n", zSql); */
       pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql);
@@ -582,6 +698,7 @@ static int cidxFilter(
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
+    sqlite3_free(zSrcList);
     cidxFreeIndex(pIdx);
     sqlite3_free(azAfter);
   }
diff --git a/manifest b/manifest
index 48fe0a91a2..59ee792f38 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions.
-D 2017-10-30T17:05:18.290
+C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check.
+D 2017-10-30T19:38:41.046
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7
+F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4
+F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
-R 45b731228881e6db68d5d1d151d2789f
+P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
+R 32cef05dfb77ed0c73cb51913be9d3e3
 U dan
-Z 8f0b667700c59bebe92a45558d3d25e5
+Z 4d91b1f15edf1ab6cd873cde5a983874
diff --git a/manifest.uuid b/manifest.uuid
index 5d8e25d480..f363d67abb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
\ No newline at end of file
+8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
index 6648187559..a91455558f 100644
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -259,6 +259,64 @@ do_index_check_test 4.3 t4cc {
   {} 'AAB','EEE',4
 }
 
+#--------------------------------------------------------------------------
+# Test an index on an expression.
+#
+do_execsql_test 5.0 {
+  CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y));
+  INSERT INTO t5 VALUES(1, '{"x":1, "y":1}');
+  INSERT INTO t5 VALUES(2, '{"x":2, "y":2}');
+  INSERT INTO t5 VALUES(3, '{"x":3, "y":3}');
+  INSERT INTO t5 VALUES(4, '{"w":4, "z":4}');
+  INSERT INTO t5 VALUES(5, '{"x":5, "y":5}');
+
+  CREATE INDEX t5x ON t5( json_extract(y, '$.x') );
+  CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC );
+}
+
+do_index_check_test 5.1.1 t5x {
+  {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5
+}
+
+do_index_check_test 5.1.2 t5y {
+  {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4
+}
+
+do_index_check_test 5.1.3 sqlite_autoindex_t5_1 {
+  {} {'{"w":4, "z":4}',4} 
+  {} {'{"x":1, "y":1}',1} 
+  {} {'{"x":2, "y":2}',2} 
+  {} {'{"x":3, "y":3}',3} 
+  {} {'{"x":5, "y":5}',5}
+}
+
+do_test 5.2 {
+  set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }]
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
+  db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);}
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
+  execsql {
+    UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1;
+    DELETE FROM xt5 WHERE rowid = 4;
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+} {}
+
+do_index_check_test 5.3.1 t5x {
+  {row missing} NULL,4 
+  {row data mismatch} 1,1 
+  {} 2,2 
+  {} 3,3 
+  {} 5,5
+}
+
+do_index_check_test 5.3.2 sqlite_autoindex_t5_1 {
+  {row missing} {'{"w":4, "z":4}',4} 
+  {row data mismatch} {'{"x":1, "y":1}',1} 
+  {} {'{"x":2, "y":2}',2} 
+  {} {'{"x":3, "y":3}',3} 
+  {} {'{"x":5, "y":5}',5}
+}
 
 
 finish_test

From 07dae088b9723550a91d492d92212f3afed89b5d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 30 Oct 2017 20:44:36 +0000
Subject: [PATCH 233/270] Clarify some comments describing the WAL index file. 
 No changes to code.

FossilOrigin-Name: 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 16 ++++++++++++++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index e88bdbcfad..0913ececd3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sperformance\sof\sthe\sLIKE\soperator\sby\susing\sstrcspn()\sto\said\nwildcard\smatching.
-D 2017-10-30T18:49:11.595
+C Clarify\ssome\scomments\sdescribing\sthe\sWAL\sindex\sfile.\s\sNo\schanges\sto\scode.
+D 2017-10-30T20:44:36.285
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
+F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 457eedfac0ff652912c4cfb89682acb3d2e372ef00745139adf8492a0b61fa09
-R b136c0336cd9a2670f5306d89a5f1d34
+P 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4
+R b73d22fe388dfec64db639a4eb4f982f
 U drh
-Z 1ff3723c7287a54f5348d686239a8395
+Z b8d196e6ebc579e3254aa8271770e0e4
diff --git a/manifest.uuid b/manifest.uuid
index e6859f5555..7d46081730 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4
\ No newline at end of file
+3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 9930b84421..0785dd3b00 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -132,6 +132,10 @@
 ** on a network filesystem.  All users of the database must be able to
 ** share memory.
 **
+** In the default unix and windows implementation, the wal-index is a mmapped
+** file whose name is the database name with a "-shm" suffix added.  For that
+** reason, the wal-index is sometimes called the "shm" file.
+**
 ** The wal-index is transient.  After a crash, the wal-index can (and should
 ** be) reconstructed from the original WAL file.  In fact, the VFS is required
 ** to either truncate or zero the header of the wal-index when the last
@@ -271,9 +275,18 @@ int sqlite3WalTrace = 0;
 #define WALINDEX_MAX_VERSION 3007000
 
 /*
-** Indices of various locking bytes.   WAL_NREADER is the number
+** Index numbers for various locking bytes.   WAL_NREADER is the number
 ** of available reader locks and should be at least 3.  The default
 ** is SQLITE_SHM_NLOCK==8 and  WAL_NREADER==5.
+**
+** Technically, the various VFSes are free to implement these locks however
+** they see fit.  However, compatibility is encouraged so that VFSes can
+** interoperate.  The standard implemention used on both unix and windows
+** is for the index number to indicate a byte offset into the
+** WalCkptInfo.aLock[] array in the wal-index header.  In other words, all
+** locks are on the shm file.  The WALINDEX_LOCK_OFFSET constant (which
+** should be 120) is the location in the shm file for the first locking
+** byte.
 */
 #define WAL_WRITE_LOCK         0
 #define WAL_ALL_BUT_WRITE      1
@@ -397,7 +410,6 @@ struct WalCkptInfo {
 #define WAL_FRAME_HDRSIZE 24
 
 /* Size of write ahead log header, including checksum. */
-/* #define WAL_HDRSIZE 24 */
 #define WAL_HDRSIZE 32
 
 /* WAL magic value. Either this value, or the same value with the least

From 666fb6914d659cf9e997dfa5d97f3124cdad39a8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 30 Oct 2017 23:25:06 +0000
Subject: [PATCH 234/270] Add the showshm utility program for printing out the
 shm header in a human-readable format.

FossilOrigin-Name: f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d
---
 Makefile.in    |   3 +
 Makefile.msc   |   3 +
 main.mk        |   3 +
 manifest       |  17 +++---
 manifest.uuid  |   2 +-
 tool/showshm.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 177 insertions(+), 9 deletions(-)
 create mode 100644 tool/showshm.c

diff --git a/Makefile.in b/Makefile.in
index 5f6be57c37..d03f009b79 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1203,6 +1203,9 @@ showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
 showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
 
+showshm$(TEXE):	$(TOP)/tool/showshm.c
+	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
diff --git a/Makefile.msc b/Makefile.msc
index 2804c25ab4..fb6f61979e 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2224,6 +2224,9 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+showshm.exe:	$(TOP)\tool\showshm.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
diff --git a/main.mk b/main.mk
index c7711fb6e9..346dd8feb0 100644
--- a/main.mk
+++ b/main.mk
@@ -908,6 +908,9 @@ showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
 		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
 
+showshm$(EXE):	$(TOP)/tool/showshm.c
+	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
+
 changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
 		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
diff --git a/manifest b/manifest
index 0913ececd3..362607f4d0 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Clarify\ssome\scomments\sdescribing\sthe\sWAL\sindex\sfile.\s\sNo\schanges\sto\scode.
-D 2017-10-30T20:44:36.285
-F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
+C Add\sthe\sshowshm\sutility\sprogram\sfor\sprinting\sout\sthe\sshm\sheader\sin\sa\nhuman-readable\sformat.
+D 2017-10-30T23:25:06.424
+F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
+F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -386,7 +386,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8
+F main.mk a6e3fa3c4d145189a67ad89e3f0d6541e8c085f5c34a21df0b1c870eb3aeb6ce
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1622,6 +1622,7 @@ F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076
 F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
+F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809
 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
@@ -1666,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 37284d4e8f501a37c582aa899419a3dfe0932e2a8e2ef2fdf59addd1d3cdacb4
-R b73d22fe388dfec64db639a4eb4f982f
+P 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7
+R 5511878351634578cf57017f9253148a
 U drh
-Z b8d196e6ebc579e3254aa8271770e0e4
+Z 5f57a6784d89256cd72e7dad10ab878c
diff --git a/manifest.uuid b/manifest.uuid
index 7d46081730..a5b1cdf01a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7
\ No newline at end of file
+f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d
\ No newline at end of file
diff --git a/tool/showshm.c b/tool/showshm.c
new file mode 100644
index 0000000000..03e0fc460c
--- /dev/null
+++ b/tool/showshm.c
@@ -0,0 +1,158 @@
+/*
+** A utility for printing content from the wal-index or "shm" file.
+*/
+#include <stdio.h>
+#include <ctype.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <assert.h>
+
+#define ISDIGIT(X)  isdigit((unsigned char)(X))
+#define ISPRINT(X)  isprint((unsigned char)(X))
+
+#if !defined(_MSC_VER)
+#include <unistd.h>
+#include <sys/types.h>
+#else
+#include <io.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+static int fd = -1;             /* The open SHM file */
+
+/* Report an out-of-memory error and die.
+*/
+static void out_of_memory(void){
+  fprintf(stderr,"Out of memory...\n");
+  exit(1);
+}
+
+/*
+** Read content from the file.
+**
+** Space to hold the content is obtained from malloc() and needs to be
+** freed by the caller.
+*/
+static unsigned char *getContent(int ofst, int nByte){
+  unsigned char *aData;
+  aData = malloc(nByte);
+  if( aData==0 ) out_of_memory();
+  lseek(fd, ofst, SEEK_SET);
+  read(fd, aData, nByte);
+  return aData;
+}
+
+/*
+** Flags values
+*/
+#define FG_HEX     1    /* Show as hex */
+#define FG_NBO     2    /* Native byte order */
+#define FG_PGSZ    4    /* Show as page-size */
+
+/* Print a line of decode output showing a 4-byte integer.
+*/
+static void print_decode_line(
+  unsigned char *aData,      /* Content being decoded */
+  int ofst, int nByte,       /* Start and size of decode */
+  unsigned flg,              /* Display flags */
+  const char *zMsg           /* Message to append */
+){
+  int i, j;
+  int val = aData[ofst];
+  char zBuf[100];
+  sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]);
+  i = (int)strlen(zBuf);
+  for(j=1; j<4; j++){
+    if( j>=nByte ){
+      sprintf(&zBuf[i], "   ");
+    }else{
+      sprintf(&zBuf[i], " %02x", aData[ofst+j]);
+      val = val*256 + aData[ofst+j];
+    }
+    i += (int)strlen(&zBuf[i]);
+  }
+  if( nByte==8 ){
+    for(j=4; j<8; j++){
+      sprintf(&zBuf[i], " %02x", aData[ofst+j]);
+      i += (int)strlen(&zBuf[i]);
+    }
+  }
+  if( flg & FG_NBO ){
+    assert( nByte==4 );
+    memcpy(&val, aData+ofst, 4);
+  }
+  sprintf(&zBuf[i], "            ");
+  i += 12;
+  if( flg & FG_PGSZ ){
+    unsigned short sz;
+    memcpy(&sz, aData+ofst, 2);
+    sprintf(&zBuf[i], "   %9d", sz==1 ? 65536 : sz);
+  }else if( flg & FG_HEX ){
+    sprintf(&zBuf[i], "  0x%08x", val);
+  }else if( nByte<8 ){
+    sprintf(&zBuf[i], "   %9d", val);
+  }
+  printf("%s  %s\n", zBuf, zMsg);
+}
+
+/*
+** Print an instance of the WalIndexHdr object.  ix is either 0 or 1
+** to select which header to print.
+*/
+static void print_index_hdr(unsigned char *aData, int ix){
+  int i;
+  assert( ix==0 || ix==1 );
+  i = ix ? 48 : 0;
+  print_decode_line(aData, 0+i, 4, FG_NBO,  "Wal-index version");
+  print_decode_line(aData, 4+i, 4, 0,       "unused padding");
+  print_decode_line(aData, 8+i, 4, FG_NBO,  "transaction counter");
+  print_decode_line(aData,12+i, 1, 0,       "1 when initialized");
+  print_decode_line(aData,13+i, 1, 0,       "true if WAL cksums are bigendian");
+  print_decode_line(aData,14+i, 2, FG_PGSZ, "database page size");
+  print_decode_line(aData,16+i, 4, FG_NBO,  "mxFrame");
+  print_decode_line(aData,20+i, 4, FG_NBO,  "Size of database in pages");
+  print_decode_line(aData,24+i, 8, 0, "Cksum of last frame in -wal");
+  print_decode_line(aData,32+i, 8, 0,  "Salt values from the -wal");
+  print_decode_line(aData,40+i, 8, 0,  "Cksum over all prior fields");
+}
+
+/*
+** Print the WalCkptInfo object
+*/
+static void print_ckpt_info(unsigned char *aData){
+  const int i = 96;
+  int j;
+  print_decode_line(aData, 0+i, 4, FG_NBO,  "nBackfill");
+  for(j=0; j<5; j++){
+    char zLabel[100];
+    sprintf(zLabel, "aReadMark[%d]", j);
+    print_decode_line(aData, 4*j+4+i, 4, FG_NBO, zLabel);
+  }
+  print_decode_line(aData,24+i, 8, 0,       "aLock");
+  print_decode_line(aData,32+i, 4, FG_NBO,  "nBackfillAttempted");
+  print_decode_line(aData,36+i, 4, FG_NBO,  "notUsed0");
+}
+
+
+int main(int argc, char **argv){
+  unsigned char *aData;
+  if( argc<2 ){
+    fprintf(stderr,"Usage: %s FILENAME\n", argv[0]);
+    exit(1);
+  }
+  fd = open(argv[1], O_RDONLY);
+  if( fd<0 ){
+    fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]);
+    exit(1);
+  }
+  aData = getContent(0, 136);
+  print_index_hdr(aData, 0);
+  print_index_hdr(aData, 1);
+  print_ckpt_info(aData);
+  free(aData);
+  close(fd);
+  return 0;
+}

From 9d9c41e25e460130678e5883fafaa13e97024add Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 03:40:15 +0000
Subject: [PATCH 235/270] Remove unnecessary NEVER() and ALWAYS() conditionals.

FossilOrigin-Name: 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce
---
 manifest        | 30 +++++++++++++++---------------
 manifest.uuid   |  2 +-
 src/build.c     |  3 ++-
 src/insert.c    |  3 ++-
 src/pcache.c    | 19 +++++++++----------
 src/prepare.c   |  3 ++-
 src/select.c    |  3 ++-
 src/vdbeaux.c   |  2 +-
 src/vdbemem.c   | 38 ++++++++++++++++++--------------------
 src/walker.c    | 19 +++++++++----------
 src/where.c     | 27 +++++++++++++--------------
 src/wherecode.c |  5 +++--
 12 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/manifest b/manifest
index 362607f4d0..b25e49185c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sshowshm\sutility\sprogram\sfor\sprinting\sout\sthe\sshm\sheader\sin\sa\nhuman-readable\sformat.
-D 2017-10-30T23:25:06.424
+C Remove\sunnecessary\sNEVER()\sand\sALWAYS()\sconditionals.
+D 2017-10-31T03:40:15.885
 F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d
@@ -408,7 +408,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c e24b61144f9c9b15c4aa05954514190361061da721e56dcd1af6e0e945ee5909
+F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -425,7 +425,7 @@ F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c 1f33ef4ca0553b60fff03aa171370f8709a3e945acfcc68ccafc92752d872f40
+F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
 F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c
@@ -453,17 +453,17 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
-F src/pcache.c 4bada070456980c3c1f16d58ec2e64e389ad77b935e3d77e0c96e7bbd397289c
+F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c 9a141a1b02dca53beaa9771699d390aafcac01f5d1f1c0ae6e23ded8dcdb709a
+F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 36345c63153e3e3e8ba533c6d3be6201c317358e8c2b34c51400abdf192515ab
+F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
 F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -536,19 +536,19 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c c423065d50cee24bc8cba57764f5e9869a1bb920c50907f5dd363ebd7c5aef82
+F src/vdbeaux.c 9521a9364e68edad3c8d05ae63395d076724bed1c878c9b13fab61ada54e7d2a
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
+F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
-F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
-F src/where.c d8c6d690c4b11f30211de073011fe19352364a6303ae053f45cb66f9576ba8a9
+F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
+F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c e8c2ece5843ea56e6c90277d421f2d628f3f7b7c976642369cc519f008e1d2b1
+F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2
 F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3be3aad9ecbe33060cfa9c6059b9206ed221d1fd72a69c355a9387f9f4e075e7
-R 5511878351634578cf57017f9253148a
+P f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d
+R 3a78a4e252d74365847a93117da52332
 U drh
-Z 5f57a6784d89256cd72e7dad10ab878c
+Z 5327e6fab7ec504e303ebb0642487352
diff --git a/manifest.uuid b/manifest.uuid
index a5b1cdf01a..0446940796 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d
\ No newline at end of file
+1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index a9428104a3..fc4dbcdba8 100644
--- a/src/build.c
+++ b/src/build.c
@@ -3852,9 +3852,10 @@ SrcList *sqlite3SrcListAppendFromTerm(
     goto append_from_error;
   }
   p = sqlite3SrcListAppend(db, p, pTable, pDatabase);
-  if( p==0 || NEVER(p->nSrc==0) ){
+  if( p==0 ){
     goto append_from_error;
   }
+  assert( p->nSrc>0 );
   pItem = &p->a[p->nSrc-1];
   assert( pAlias!=0 );
   if( pAlias->n ){
diff --git a/src/insert.c b/src/insert.c
index 5da52e7692..3a531495cf 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -909,7 +909,8 @@ void sqlite3Insert(
         VdbeOp *pOp;
         sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid);
         pOp = sqlite3VdbeGetOp(v, -1);
-        if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){
+        assert( pOp!=0 );
+        if( pOp->opcode==OP_Null && !IsVirtual(pTab) ){
           appendFlag = 1;
           pOp->opcode = OP_NewRowid;
           pOp->p1 = iDataCur;
diff --git a/src/pcache.c b/src/pcache.c
index 6f43a1e8af..4b2e481c27 100644
--- a/src/pcache.c
+++ b/src/pcache.c
@@ -566,16 +566,15 @@ void sqlite3PcacheMakeDirty(PgHdr *p){
 */
 void sqlite3PcacheMakeClean(PgHdr *p){
   assert( sqlite3PcachePageSanity(p) );
-  if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){
-    assert( (p->flags & PGHDR_CLEAN)==0 );
-    pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
-    p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
-    p->flags |= PGHDR_CLEAN;
-    pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno));
-    assert( sqlite3PcachePageSanity(p) );
-    if( p->nRef==0 ){
-      pcacheUnpin(p);
-    }
+  assert( (p->flags & PGHDR_DIRTY)!=0 );
+  assert( (p->flags & PGHDR_CLEAN)==0 );
+  pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
+  p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
+  p->flags |= PGHDR_CLEAN;
+  pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno));
+  assert( sqlite3PcachePageSanity(p) );
+  if( p->nRef==0 ){
+    pcacheUnpin(p);
   }
 }
 
diff --git a/src/prepare.c b/src/prepare.c
index 8ff27fec6c..9141cb8a78 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -473,7 +473,8 @@ int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){
   */
   assert( sqlite3_mutex_held(db->mutex) );
   if( pSchema ){
-    for(i=0; ALWAYS(i<db->nDb); i++){
+    for(i=0; 1; i++){
+      assert( i<db->nDb );
       if( db->aDb[i].pSchema==pSchema ){
         break;
       }
diff --git a/src/select.c b/src/select.c
index f31dd850ab..46c69d71fb 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4342,7 +4342,8 @@ static int selectExpander(Walker *pWalker, Select *p){
   if( db->mallocFailed  ){
     return WRC_Abort;
   }
-  if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){
+  assert( p->pSrc!=0 );
+  if( (selFlags & SF_Expanded)!=0 ){
     return WRC_Prune;
   }
   pTabList = p->pSrc;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 6ec8b30680..e057a5df5f 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3043,7 +3043,7 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
 void sqlite3VdbeDelete(Vdbe *p){
   sqlite3 *db;
 
-  if( NEVER(p==0) ) return;
+  assert( p!=0 );
   db = p->db;
   assert( sqlite3_mutex_held(db->mutex) );
   sqlite3VdbeClearObject(db, p);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 3c07f5a1a3..f9f58c43f4 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -350,26 +350,24 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){
 ** otherwise.
 */
 int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){
-  int rc = SQLITE_OK;
-  if( ALWAYS(pFunc && pFunc->xFinalize) ){
-    sqlite3_context ctx;
-    Mem t;
-    assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef );
-    assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
-    memset(&ctx, 0, sizeof(ctx));
-    memset(&t, 0, sizeof(t));
-    t.flags = MEM_Null;
-    t.db = pMem->db;
-    ctx.pOut = &t;
-    ctx.pMem = pMem;
-    ctx.pFunc = pFunc;
-    pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */
-    assert( (pMem->flags & MEM_Dyn)==0 );
-    if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc);
-    memcpy(pMem, &t, sizeof(t));
-    rc = ctx.isError;
-  }
-  return rc;
+  sqlite3_context ctx;
+  Mem t;
+  assert( pFunc!=0 );
+  assert( pFunc->xFinalize!=0 );
+  assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef );
+  assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
+  memset(&ctx, 0, sizeof(ctx));
+  memset(&t, 0, sizeof(t));
+  t.flags = MEM_Null;
+  t.db = pMem->db;
+  ctx.pOut = &t;
+  ctx.pMem = pMem;
+  ctx.pFunc = pFunc;
+  pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */
+  assert( (pMem->flags & MEM_Dyn)==0 );
+  if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc);
+  memcpy(pMem, &t, sizeof(t));
+  return ctx.isError;
 }
 
 /*
diff --git a/src/walker.c b/src/walker.c
index 49f9fa1897..ae7545bf5f 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -108,16 +108,15 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){
   struct SrcList_item *pItem;
 
   pSrc = p->pSrc;
-  if( ALWAYS(pSrc) ){
-    for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
-      if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){
-        return WRC_Abort;
-      }
-      if( pItem->fg.isTabFunc
-       && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg)
-      ){
-        return WRC_Abort;
-      }
+  assert( pSrc!=0 );
+  for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
+    if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){
+      return WRC_Abort;
+    }
+    if( pItem->fg.isTabFunc
+     && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg)
+    ){
+      return WRC_Abort;
     }
   }
   return WRC_Continue;
diff --git a/src/where.c b/src/where.c
index 6021edf47f..4c1c332d2a 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1863,22 +1863,21 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
 ** Free a WhereInfo structure
 */
 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
-  if( ALWAYS(pWInfo) ){
-    int i;
-    for(i=0; i<pWInfo->nLevel; i++){
-      WhereLevel *pLevel = &pWInfo->a[i];
-      if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){
-        sqlite3DbFree(db, pLevel->u.in.aInLoop);
-      }
+  int i;
+  assert( pWInfo!=0 );
+  for(i=0; i<pWInfo->nLevel; i++){
+    WhereLevel *pLevel = &pWInfo->a[i];
+    if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){
+      sqlite3DbFree(db, pLevel->u.in.aInLoop);
     }
-    sqlite3WhereClauseClear(&pWInfo->sWC);
-    while( pWInfo->pLoops ){
-      WhereLoop *p = pWInfo->pLoops;
-      pWInfo->pLoops = p->pNextLoop;
-      whereLoopDelete(db, p);
-    }
-    sqlite3DbFreeNN(db, pWInfo);
   }
+  sqlite3WhereClauseClear(&pWInfo->sWC);
+  while( pWInfo->pLoops ){
+    WhereLoop *p = pWInfo->pLoops;
+    pWInfo->pLoops = p->pNextLoop;
+    whereLoopDelete(db, p);
+  }
+  sqlite3DbFreeNN(db, pWInfo);
 }
 
 /*
diff --git a/src/wherecode.c b/src/wherecode.c
index 6cdf7b566b..da5c686a95 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -294,8 +294,8 @@ void sqlite3WhereAddScanStatus(
 */
 static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){
   int nLoop = 0;
-  while( ALWAYS(pTerm!=0)
-      && (pTerm->wtFlags & TERM_CODED)==0
+  assert( pTerm!=0 );
+  while( (pTerm->wtFlags & TERM_CODED)==0
       && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin))
       && (pLevel->notReady & pTerm->prereqAll)==0
   ){
@@ -306,6 +306,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){
     }
     if( pTerm->iParent<0 ) break;
     pTerm = &pTerm->pWC->a[pTerm->iParent];
+    assert( pTerm!=0 );
     pTerm->nChild--;
     if( pTerm->nChild!=0 ) break;
     nLoop++;

From 7eded5925b89222b4934a36a8c3d118dee4d7457 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 31 Oct 2017 12:01:01 +0000
Subject: [PATCH 236/270] Fix a couple of issues in incremental_index_check to
 do with CREATE INDEX statements that contain embedded SQL comments.

FossilOrigin-Name: 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8
---
 ext/repair/checkindex.c | 146 ++++++++++++++++++++++++----------------
 manifest                |  14 ++--
 manifest.uuid           |   2 +-
 test/checkindex.test    |  27 ++++++++
 4 files changed, 123 insertions(+), 66 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index fc91b74244..16fb3b327e 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -274,54 +274,6 @@ static void cidxFreeIndex(CidxIndex *pIdx){
   }
 }
 
-#define CIDX_PARSE_EOF   0
-#define CIDX_PARSE_COMMA 1      /*  "," */
-#define CIDX_PARSE_OPEN  2      /*  "(" */
-#define CIDX_PARSE_CLOSE 3      /*  ")" */
-
-static int cidxFindNext(const char *zIn, const char **pzOut){
-  const char *z = zIn;
-
-  while( 1 ){
-    *pzOut = z;
-    switch( *z ){
-      case '\0':
-        return CIDX_PARSE_EOF;
-      case '(':
-        return CIDX_PARSE_OPEN;
-      case ')':
-        return CIDX_PARSE_CLOSE;
-      case ',':
-        return CIDX_PARSE_COMMA;
-
-      case '"': 
-      case '\'': 
-      case '`': {
-        char q = *z;
-        z++;
-        while( *z ){
-          if( *z==q ){
-            z++;
-            if( *z!=q ) break;
-          }
-          z++;
-        }
-        break;
-      }
-
-      case '[':
-        while( *z++!=']' );
-        break;
-
-      default:
-        z++;
-    }
-  }
-
-  assert( 0 );
-  return -1;
-}
-
 static int cidx_isspace(char c){
   return c==' ' || c=='\t' || c=='\r' || c=='\n';
 }
@@ -332,36 +284,114 @@ static int cidx_isident(char c){
     || (c>='A' && c<='Z') || c=='_';
 }
 
+#define CIDX_PARSE_EOF   0
+#define CIDX_PARSE_COMMA 1      /*  "," */
+#define CIDX_PARSE_OPEN  2      /*  "(" */
+#define CIDX_PARSE_CLOSE 3      /*  ")" */
+
+static int cidxFindNext(
+  const char *zIn, 
+  const char **pzOut,
+  int *pbDoNotTrim                /* OUT: True if prev is -- comment */
+){
+  const char *z = zIn;
+
+  while( 1 ){
+    if( z[0]=='-' && z[1]=='-' ){
+      z += 2;
+      while( z[0]!='\n' ){
+        if( z[0]=='\0' ) return CIDX_PARSE_EOF;
+        z++;
+      }
+      while( cidx_isspace(*z) ) z++;
+      *pbDoNotTrim = 1;
+    }else{
+      *pzOut = z;
+      switch( *z ){
+        case '\0':
+          return CIDX_PARSE_EOF;
+        case '(':
+          return CIDX_PARSE_OPEN;
+        case ')':
+          return CIDX_PARSE_CLOSE;
+        case ',':
+          return CIDX_PARSE_COMMA;
+  
+        case '"': 
+        case '\'': 
+        case '`': {
+          char q = *z;
+          z++;
+          while( *z ){
+            if( *z==q ){
+              z++;
+              if( *z!=q ) break;
+            }
+            z++;
+          }
+          break;
+        }
+  
+        case '[':
+          while( *z++!=']' );
+          break;
+  
+        case '/':
+          if( z[1]=='*' ){
+            z += 2;
+            while( z[0]!='*' || z[1]!='/' ){
+              if( z[1]=='\0' ) return CIDX_PARSE_EOF;
+              z++;
+            }
+            z += 2;
+            break;
+          }
+  
+        default:
+          z++;
+          break;
+      }
+      *pbDoNotTrim = 0;
+    }
+  }
+
+  assert( 0 );
+  return -1;
+}
+
 static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){
   const char *z = zSql;
   const char *z1;
   int e;
   int rc = SQLITE_OK;
   int nParen = 1;
+  int bDoNotTrim = 0;
   CidxColumn *pCol = pIdx->aCol;
 
-  e = cidxFindNext(z, &z);
+  e = cidxFindNext(z, &z, &bDoNotTrim);
   if( e!=CIDX_PARSE_OPEN ) goto parse_error;
   z1 = z+1;
   z++;
   while( nParen>0 ){
-    e = cidxFindNext(z, &z);
+    e = cidxFindNext(z, &z, &bDoNotTrim);
     if( e==CIDX_PARSE_EOF ) goto parse_error;
     if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){
       const char *z2 = z;
       if( pCol->zExpr ) goto parse_error;
 
-      while( cidx_isspace(z[-1]) ) z--;
-      if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){
-        z -= 3;
-        while( cidx_isspace(z[-1]) ) z--;
-      }else
-      if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){
-        z -= 4;
+      if( bDoNotTrim==0 ){
         while( cidx_isspace(z[-1]) ) z--;
+        if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){
+          z -= 3;
+          while( cidx_isspace(z[-1]) ) z--;
+        }else
+          if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){
+            z -= 4;
+            while( cidx_isspace(z[-1]) ) z--;
+          }
+        while( cidx_isspace(z1[0]) ) z1++;
       }
 
-      while( cidx_isspace(z1[0]) ) z1++;
       pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1);
       pCol++;
       z = z1 = z2+1;
diff --git a/manifest b/manifest
index 59ee792f38..974a630385 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check.
-D 2017-10-30T19:38:41.046
+C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments.
+D 2017-10-31T12:01:01.861
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185
+F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3
+F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
-R 32cef05dfb77ed0c73cb51913be9d3e3
+P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
+R 4a5b38f6ae42388faee270886368a889
 U dan
-Z 4d91b1f15edf1ab6cd873cde5a983874
+Z ee7435bc6b4be93625562670a465a6a7
diff --git a/manifest.uuid b/manifest.uuid
index f363d67abb..be302e7d98 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
\ No newline at end of file
+2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8
\ No newline at end of file
diff --git a/test/checkindex.test b/test/checkindex.test
index a91455558f..22463d3ffe 100644
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -318,6 +318,33 @@ do_index_check_test 5.3.2 sqlite_autoindex_t5_1 {
   {} {'{"x":5, "y":5}',5}
 }
 
+#-------------------------------------------------------------------------
+#
+do_execsql_test 6.0 {
+  CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z);
+  CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z);
+  CREATE INDEX t6x2 ON t6(z, -- hello,world,
+  y);
+
+  CREATE INDEX t6x3 ON t6(z -- hello,world
+  , y);
+
+  INSERT INTO t6 VALUES(1, 2, 3);
+  INSERT INTO t6 VALUES(4, 5, 6);
+}
+
+do_index_check_test 6.1 t6x1 {
+  {} 2,3,1 
+  {} 5,6,4
+}
+do_index_check_test 6.2 t6x2 {
+  {} 3,2,1 
+  {} 6,5,4
+}
+do_index_check_test 6.2 t6x3 {
+  {} 3,2,1 
+  {} 6,5,4
+}
 
 finish_test
 

From 183f0aa6c4a72401ce4a00dc94f5f63744f10f6b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 12:06:29 +0000
Subject: [PATCH 237/270] Fix a minor comment typo.  No changes to code.

FossilOrigin-Name: 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b25e49185c..fe7d2f74fe 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sunnecessary\sNEVER()\sand\sALWAYS()\sconditionals.
-D 2017-10-31T03:40:15.885
+C Fix\sa\sminor\scomment\stypo.\s\sNo\schanges\sto\scode.
+D 2017-10-31T12:06:29.439
 F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 712947ad0e37ccb852c631527fd708cc0dea8b6fd503c8fd070370aae37bbe56
+F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f6304fd142c998aba44f02c6018223af2630671b4791d750b70a59ab1adb8d6d
-R 3a78a4e252d74365847a93117da52332
+P 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce
+R bec87929117a677c4c0a889fc21edace
 U drh
-Z 5327e6fab7ec504e303ebb0642487352
+Z 4a41196ea5bc7a7d68e6bb73582e1035
diff --git a/manifest.uuid b/manifest.uuid
index 0446940796..cec3b4159e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce
\ No newline at end of file
+5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 0785dd3b00..19c9ea0a08 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2154,7 +2154,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 ** checkpointed.  If useWal==0 then this routine calls walIndexReadHdr() 
 ** to make a copy of the wal-index header into pWal->hdr.  If the 
 ** wal-index header has changed, *pChanged is set to 1 (as an indication 
-** to the caller that the local paget cache is obsolete and needs to be 
+** to the caller that the local page cache is obsolete and needs to be 
 ** flushed.)  When useWal==1, the wal-index header is assumed to already
 ** be loaded and the pChanged parameter is unused.
 **

From 6f099107791da2e83cd246e0ef248f4dfeb8f448 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 12:20:43 +0000
Subject: [PATCH 238/270] Fix redundancies in the makefiles that resulted from
 moving shell.c over to shell.c.in.

FossilOrigin-Name: bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f
---
 Makefile.in   |  2 +-
 Makefile.msc  |  1 -
 main.mk       |  2 +-
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index d03f009b79..7fef9f575b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -676,7 +676,7 @@ mptest:	mptester$(TEXE)
 
 sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 
 sqlite3ext.h:	.target_source
diff --git a/Makefile.msc b/Makefile.msc
index fb6f61979e..d663b753bf 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1649,7 +1649,6 @@ mptest:	mptester.exe
 
 sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
 	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
-	copy tsrc\shell.c .
 	copy $(TOP)\ext\session\sqlite3session.h .
 
 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
diff --git a/main.mk b/main.mk
index 346dd8feb0..c74759b9cc 100644
--- a/main.mk
+++ b/main.mk
@@ -582,7 +582,7 @@ target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
 
 sqlite3.c:	target_source $(TOP)/tool/mksqlite3c.tcl
 	tclsh $(TOP)/tool/mksqlite3c.tcl
-	cp tsrc/shell.c tsrc/sqlite3ext.h .
+	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
 	cat sqlite3.c >>tclsqlite3.c
diff --git a/manifest b/manifest
index fe7d2f74fe..f8ccb2f8d9 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sa\sminor\scomment\stypo.\s\sNo\schanges\sto\scode.
-D 2017-10-31T12:06:29.439
-F Makefile.in 5fb6750646b432cb72d1aa91f9a7888eb488b3de145024cc48ec9815dc106fbd
+C Fix\sredundancies\sin\sthe\smakefiles\sthat\sresulted\sfrom\smoving\sshell.c\sover\nto\sshell.c.in.
+D 2017-10-31T12:20:43.201
+F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 74ccbe1c06de753a6a0fa0fad4e7f8bea37f1ba4303448300bda3427efb64f2d
+F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -386,7 +386,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a6e3fa3c4d145189a67ad89e3f0d6541e8c085f5c34a21df0b1c870eb3aeb6ce
+F main.mk 4fafbf2c23268ad5c497bd7c9da99dd3a53b1ffa2c150b6eb975397d4bb3f12f
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1c80c75d4be2f3d44fb18bb4c07eccac2aba79b688215a741317dfc47dc7c2ce
-R bec87929117a677c4c0a889fc21edace
+P 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e
+R 2e8424d9fbc533001e55f43737738a2c
 U drh
-Z 4a41196ea5bc7a7d68e6bb73582e1035
+Z 784be9d7f67c93467ca8ada4e99f0e81
diff --git a/manifest.uuid b/manifest.uuid
index cec3b4159e..c8016ca992 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e
\ No newline at end of file
+bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f
\ No newline at end of file

From d59fde3da55e5b992030b5ed1a45ab921295458f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 14:56:44 +0000
Subject: [PATCH 239/270] Update the built procedures for the sqlite3_analyzer
 utility to allow it to be linked with an external sqlite3.o library. 
 Automatically detect a missing dbstat extension and report the compile-time
 error.

FossilOrigin-Name: bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de
---
 manifest                   | 14 +++++++-------
 manifest.uuid              |  2 +-
 tool/spaceanal.tcl         | 17 +++++++++++++++--
 tool/sqlite3_analyzer.c.in |  2 ++
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index f8ccb2f8d9..649893539e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sredundancies\sin\sthe\smakefiles\sthat\sresulted\sfrom\smoving\sshell.c\sover\nto\sshell.c.in.
-D 2017-10-31T12:20:43.201
+C Update\sthe\sbuilt\sprocedures\sfor\sthe\ssqlite3_analyzer\sutility\sto\sallow\sit\nto\sbe\slinked\swith\san\sexternal\ssqlite3.o\slibrary.\s\sAutomatically\sdetect\sa\nmissing\sdbstat\sextension\sand\sreport\sthe\scompile-time\serror.
+D 2017-10-31T14:56:44.588
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -1626,7 +1626,7 @@ F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a80
 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
-F tool/spaceanal.tcl f40dc82b4d5e39d040a02a3ec38268e324068815e4292a15ffa30ee93208bbfd
+F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec
 F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
@@ -1635,7 +1635,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
-F tool/sqlite3_analyzer.c.in 771d15fb9c67645fd8ef932a438f98959da4b7c7da3cb87ae1850b27c969edf3
+F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1667,7 +1667,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5f79e6d9c3d44b0cc7cf805429006f7b29a69e8d863b58472172a56b29a7bb4e
-R 2e8424d9fbc533001e55f43737738a2c
+P bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f
+R 405d6881e4ae5254162d311375149a09
 U drh
-Z 784be9d7f67c93467ca8ada4e99f0e81
+Z b66d68a4e4b6a1ef52955c34a93fec95
diff --git a/manifest.uuid b/manifest.uuid
index c8016ca992..45e7b5b385 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf09fa683ea42b7552bd2f29ab5371bd175f0055cf9a453e6f8c6f4408cd168f
\ No newline at end of file
+bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de
\ No newline at end of file
diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl
index 9db108dc26..3e08d3ffa7 100644
--- a/tool/spaceanal.tcl
+++ b/tool/spaceanal.tcl
@@ -1,7 +1,9 @@
-# Run this TCL script using "testfixture" in order get a report that shows
-# how much disk space is used by a particular data to actually store data
+# Run this TCL script using an SQLite-enabled TCL interpreter to get a report
+# on how much disk space is used by a particular data to actually store data
 # versus how much space is unused.
 #
+# The dbstat virtual table is required.
+#
 
 if {[catch {
 
@@ -147,6 +149,17 @@ if {$flags(-debug)} {
   db trace ::dbtrace
 }
 
+# Make sure all required compile-time options are available
+#
+if {![db exists {SELECT 1 FROM pragma_compile_options
+                WHERE compile_options='ENABLE_DBSTAT_VTAB'}]} {
+  puts "The SQLite database engine linked with this application\
+        lacks required capabilities. Recompile using the\
+        -DSQLITE_ENABLE_DBSTAT_VTAB compile-time option to fix\
+        this problem."
+  exit 1
+}
+
 db eval {SELECT count(*) FROM sqlite_master}
 set pageSize [expr {wide([db one {PRAGMA page_size}])}]
 
diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in
index 547b22ffe1..e830521e08 100644
--- a/tool/sqlite3_analyzer.c.in
+++ b/tool/sqlite3_analyzer.c.in
@@ -14,7 +14,9 @@
 #define SQLITE_DEFAULT_MEMSTATUS 0
 #define SQLITE_MAX_EXPR_DEPTH 0
 #define SQLITE_OMIT_LOAD_EXTENSION 1
+#ifndef USE_EXTERNAL_SQLITE
 INCLUDE sqlite3.c
+#endif
 INCLUDE $ROOT/src/tclsqlite.c
 
 const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){

From 2737fa037660b4694ff5c190f9de955c15673d47 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 15:49:19 +0000
Subject: [PATCH 240/270] Add build infrastructure for the sqlite3_checker
 command-line utility.

FossilOrigin-Name: 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09
---
 Makefile.in                     | 15 ++++++
 Makefile.msc                    | 17 +++++++
 ext/misc/btreeinfo.c            |  4 +-
 ext/repair/sqlite3_checker.c.in | 31 ++++++++++++
 ext/repair/sqlite3_checker.tcl  | 88 +++++++++++++++++++++++++++++++++
 main.mk                         | 16 ++++++
 manifest                        | 22 +++++----
 manifest.uuid                   |  2 +-
 8 files changed, 183 insertions(+), 12 deletions(-)
 create mode 100644 ext/repair/sqlite3_checker.c.in
 create mode 100644 ext/repair/sqlite3_checker.tcl

diff --git a/Makefile.in b/Makefile.in
index 5f6be57c37..74ade8241e 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1187,6 +1187,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
+
+sqlite3_checker$(TEXE):	sqlite3_checker.c
+	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
+
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
diff --git a/Makefile.msc b/Makefile.msc
index 2804c25ab4..1e069cfe66 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1491,6 +1491,7 @@ TESTPROGS = \
   testfixture.exe \
   $(SQLITE3EXE) \
   sqlite3_analyzer.exe \
+  sqlite3_checker.exe \
   sqldiff.exe \
   dbhash.exe
 
@@ -2198,6 +2199,22 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
 
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@
+
+sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c
index 131c63e835..d75f06277f 100644
--- a/ext/misc/btreeinfo.c
+++ b/ext/misc/btreeinfo.c
@@ -61,7 +61,9 @@
 **      SELECT name FROM sqlite_btreeinfo
 **       WHERE type='table' AND NOT hasRowid;
 */
-#include <sqlite3ext.h>
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
 SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>
diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in
new file mode 100644
index 0000000000..af8b673ec5
--- /dev/null
+++ b/ext/repair/sqlite3_checker.c.in
@@ -0,0 +1,31 @@
+/*
+** Read an SQLite database file and analyze its space utilization.  Generate
+** text on standard output.
+*/
+#define TCLSH_INIT_PROC sqlite3_checker_init_proc
+#define SQLITE_ENABLE_DBPAGE_VTAB 1
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 0
+#undef SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_OMIT_DECLTYPE 1
+#define SQLITE_OMIT_DEPRECATED 1
+#define SQLITE_OMIT_PROGRESS_CALLBACK 1
+#define SQLITE_OMIT_SHARED_CACHE 1
+#define SQLITE_DEFAULT_MEMSTATUS 0
+#define SQLITE_MAX_EXPR_DEPTH 0
+INCLUDE sqlite3.c
+INCLUDE $ROOT/src/tclsqlite.c
+INCLUDE $ROOT/ext/misc/btreeinfo.c
+INCLUDE $ROOT/ext/repair/checkindex.c
+
+#include <stdio.h>
+
+const char *sqlite3_checker_init_proc(Tcl_Interp *interp){
+  sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init);
+  sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init);
+  return
+BEGIN_STRING
+INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl
+END_STRING
+;
+}
diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
new file mode 100644
index 0000000000..86799741bc
--- /dev/null
+++ b/ext/repair/sqlite3_checker.tcl
@@ -0,0 +1,88 @@
+# Read and run TCL commands from standard input.  Used to implement
+# the --tclsh option.
+# This TCL script is the main driver script for the sqlite3_checker utility
+# program.
+#
+
+proc tclsh {} {
+  set line {}
+  while {![eof stdin]} {
+    if {$line!=""} {
+      puts -nonewline "> "
+    } else {
+      puts -nonewline "% "
+    }
+    flush stdout
+    append line [gets stdin]
+    if {[info complete $line]} {
+      if {[catch {uplevel #0 $line} result]} {
+        puts stderr "Error: $result"
+      } elseif {$result!=""} {
+        puts $result
+      }
+      set line {}
+    } else {
+      append line \n
+    }
+  }
+}
+
+proc usage {} {
+  set argv0 [file rootname [file tail [info nameofexecutable]]]
+  puts stderr "Usage: $argv0 OPTIONS database-filename"
+  puts stderr {
+Do sanity checking on a live SQLite3 database file specified by the
+"database-filename" argument.
+
+Options:
+
+   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
+
+   --version    Show the version number of SQLite
+}
+  exit 1
+}
+
+set file_to_analyze {}
+append argv {}
+foreach arg $argv {
+  if {[regexp {^-+tclsh$} $arg]} {
+    tclsh
+    exit 0
+  }
+  if {[regexp {^-+version$} $arg]} {
+    sqlite3 mem :memory:
+    puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
+    mem close
+    exit 0
+  }
+  if {[regexp {^-} $arg]} {
+    puts stderr "Unknown option: $arg"
+    usage
+  }
+  if {$file_to_analyze!=""} {
+    usage
+  } else {
+    set file_to_analyze $arg
+  }
+}
+if {$file_to_analyze==""} usage
+
+# If a TCL script is specified on the command-line, then run that
+# script.
+#
+if {[file extension $file_to_analyze]==".tcl"} {
+  source $file_to_analyze
+  exit 0
+}
+
+set root_filename $file_to_analyze
+regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename
+if {![file exists $root_filename]} {
+  puts stderr "No such file: $root_filename"
+  exit 1
+}
+if {![file readable $root_filename]} {
+  puts stderr "File is not readable: $root_filename"
+  exit 1
+}
diff --git a/main.mk b/main.mk
index c7711fb6e9..b9072a8dcc 100644
--- a/main.mk
+++ b/main.mk
@@ -464,6 +464,7 @@ TESTPROGS = \
   testfixture$(EXE) \
   sqlite3$(EXE) \
   sqlite3_analyzer$(EXE) \
+  sqlite3_checker$(EXE) \
   sqldiff$(EXE) \
   dbhash$(EXE)
 
@@ -786,6 +787,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+CHECKER_DEPS =\
+  $(TOP)/tool/mkccode.tcl \
+  sqlite3.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
+
+sqlite3_checker.c:	$(CHECKER_DEPS)
+	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
+
+sqlite3_checker$(TEXE):	sqlite3_checker.c
+	$(TCCX) $(TCL_FLAGS) sqlite3_checker.c -o $@ $(LIBTCL) $(THREADLIB)
+
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
             $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB)
diff --git a/manifest b/manifest
index 974a630385..b28e60b264 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments.
-D 2017-10-31T12:01:01.861
-F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
+C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility.
+D 2017-10-31T15:49:19.426
+F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
+F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -258,7 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516
+F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
@@ -329,6 +329,8 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
+F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92
+F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -387,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8
+F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1668,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
-R 4a5b38f6ae42388faee270886368a889
-U dan
-Z ee7435bc6b4be93625562670a465a6a7
+P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8
+R 0fe5d7d5728508c17e7a673dce852490
+U drh
+Z 5b634bacf5fe6d374b5cc3f0a57d52b2
diff --git a/manifest.uuid b/manifest.uuid
index be302e7d98..58aa8b845a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8
\ No newline at end of file
+429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09
\ No newline at end of file

From 2091f0fb3e52c493315796185b2329de6f55cfc6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 15:55:48 +0000
Subject: [PATCH 241/270] Fix an error in the Makefile.msc for the
 sqlite3_checker.exe target.

FossilOrigin-Name: 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0
---
 Makefile.msc  |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 1e069cfe66..6d3f7c7ccd 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2209,7 +2209,7 @@ CHECKER_DEPS =\
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
 sqlite3_checker.c:	$(CHECKER_DEPS)
-	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
 
 sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
diff --git a/manifest b/manifest
index b28e60b264..20118f2662 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility.
-D 2017-10-31T15:49:19.426
+C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget.
+D 2017-10-31T15:55:48.444
 F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0
+F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8
-R 0fe5d7d5728508c17e7a673dce852490
+P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09
+R d3e35133a6b7ed531196c48b12a56714
 U drh
-Z 5b634bacf5fe6d374b5cc3f0a57d52b2
+Z 22a9f7b25cabce39b5f911e9801c9b67
diff --git a/manifest.uuid b/manifest.uuid
index 58aa8b845a..c6585cbdd7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09
\ No newline at end of file
+49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0
\ No newline at end of file

From 927dd51fce9e80feb2b3bda48bf9f77724cf0f89 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 31 Oct 2017 18:09:40 +0000
Subject: [PATCH 242/270] Add the checkfreelist extension to the
 sqlite3_checker binary.

FossilOrigin-Name: 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a
---
 Makefile.in                     |  1 +
 Makefile.msc                    |  1 +
 ext/repair/sqlite3_checker.c.in |  2 ++
 main.mk                         |  1 +
 manifest                        | 18 +++++++++---------
 manifest.uuid                   |  2 +-
 6 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 74ade8241e..0653ae8eae 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1193,6 +1193,7 @@ CHECKER_DEPS =\
   $(TOP)/src/tclsqlite.c \
   $(TOP)/ext/repair/sqlite3_checker.tcl \
   $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
   $(TOP)/ext/misc/btreeinfo.c \
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
diff --git a/Makefile.msc b/Makefile.msc
index 6d3f7c7ccd..320dacc411 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2205,6 +2205,7 @@ CHECKER_DEPS =\
   $(TOP)/src/tclsqlite.c \
   $(TOP)/ext/repair/sqlite3_checker.tcl \
   $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
   $(TOP)/ext/misc/btreeinfo.c \
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in
index af8b673ec5..6045c36a9d 100644
--- a/ext/repair/sqlite3_checker.c.in
+++ b/ext/repair/sqlite3_checker.c.in
@@ -17,12 +17,14 @@ INCLUDE sqlite3.c
 INCLUDE $ROOT/src/tclsqlite.c
 INCLUDE $ROOT/ext/misc/btreeinfo.c
 INCLUDE $ROOT/ext/repair/checkindex.c
+INCLUDE $ROOT/ext/repair/checkfreelist.c
 
 #include <stdio.h>
 
 const char *sqlite3_checker_init_proc(Tcl_Interp *interp){
   sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init);
   sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init);
+  sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init);
   return
 BEGIN_STRING
 INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl
diff --git a/main.mk b/main.mk
index b9072a8dcc..266328ac8b 100644
--- a/main.mk
+++ b/main.mk
@@ -793,6 +793,7 @@ CHECKER_DEPS =\
   $(TOP)/src/tclsqlite.c \
   $(TOP)/ext/repair/sqlite3_checker.tcl \
   $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
   $(TOP)/ext/misc/btreeinfo.c \
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
diff --git a/manifest b/manifest
index 20118f2662..9b4d6af191 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget.
-D 2017-10-31T15:55:48.444
-F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5
+C Add\sthe\scheckfreelist\sextension\sto\sthe\ssqlite3_checker\sbinary.
+D 2017-10-31T18:09:40.959
+F Makefile.in 750ff35a98dd7f4ad585923d88f4309bb4cebb51c4a6622a32f134a138c9bd13
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1
+F Makefile.msc 038f2c990966e6503f4960da574d4eef7b07c272a480983e10c4ebaa6d98cff9
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
-F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92
+F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
 F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
@@ -389,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030
+F main.mk 4f59e62614c9e2d793cdafa25f7b24f357b48b009099185c66c52af4e31ee36f
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09
-R d3e35133a6b7ed531196c48b12a56714
+P 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0
+R 0e924f8968e62617bb612a72ff226c2d
 U drh
-Z 22a9f7b25cabce39b5f911e9801c9b67
+Z 18789d1f0a3682c9ed4749672ea58f7e
diff --git a/manifest.uuid b/manifest.uuid
index c6585cbdd7..99d2299f3b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0
\ No newline at end of file
+0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a
\ No newline at end of file

From 4e8ad3bc1435e5202d1a59a9dd3e7eabfaff9469 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 00:10:34 +0000
Subject: [PATCH 243/270] Begin putting functionality into the sqlite3_checker
 binary.

FossilOrigin-Name: e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659
---
 ext/repair/sqlite3_checker.tcl | 57 ++++++++++++++++++++++++++++++++++
 manifest                       | 12 +++----
 manifest.uuid                  |  2 +-
 3 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index 86799741bc..fa3ffd73ce 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -4,6 +4,21 @@
 # program.
 #
 
+# Special case:
+#
+#      sqlite3_checker --test FILENAME ARGS
+#
+# uses FILENAME in place of this script.
+#
+if {[lindex $argv 0]=="--test" && [llength $argv]>2} {
+  set file [lindex $argv 1]
+  set argv [lrange $argv 2 end]
+  source $file
+  exit 0
+}
+
+# Emulate a TCL shell
+#
 proc tclsh {} {
   set line {}
   while {![eof stdin]} {
@@ -27,6 +42,8 @@ proc tclsh {} {
   }
 }
 
+# Print a usage message on standard error, then quit.
+#
 proc usage {} {
   set argv0 [file rootname [file tail [info nameofexecutable]]]
   puts stderr "Usage: $argv0 OPTIONS database-filename"
@@ -36,6 +53,8 @@ Do sanity checking on a live SQLite3 database file specified by the
 
 Options:
 
+   --freelist   Perform a freelist check
+
    --tclsh      Run the built-in TCL interpreter interactively (for debugging)
 
    --version    Show the version number of SQLite
@@ -45,6 +64,8 @@ Options:
 
 set file_to_analyze {}
 append argv {}
+set bFreelistCheck 0
+set bSummary 1
 foreach arg $argv {
   if {[regexp {^-+tclsh$} $arg]} {
     tclsh
@@ -56,6 +77,11 @@ foreach arg $argv {
     mem close
     exit 0
   }
+  if {[regexp {^-+freelist$} $arg]} {
+    set bFreelistCheck 1
+    set bSummary 0
+    continue
+  }
   if {[regexp {^-} $arg]} {
     puts stderr "Unknown option: $arg"
     usage
@@ -86,3 +112,34 @@ if {![file readable $root_filename]} {
   puts stderr "File is not readable: $root_filename"
   exit 1
 }
+
+if {[catch {sqlite3 db $file_to_analyze} res]} {
+  puts stderr "Cannot open datababase $root_filename: $res"
+  exit 1
+}
+
+if {$bFreelistCheck} {
+  puts "freelist-check:"
+  flush stdout
+  puts [db one {SELECT checkfreelist('main')}]
+}
+if {$bSummary} {
+  set scale 0
+  set pgsz [db one {PRAGMA page_size}]
+  db eval {SELECT nPage*$pgsz AS sz, name, tbl_name
+             FROM sqlite_btreeinfo
+            WHERE type='index'
+            ORDER BY 1 DESC, name} {
+    if {$scale==0} {
+      if {$sz>10000000} {
+        set scale 1000000.0
+        set unit MB
+      } else {
+        set scale 1000.0
+        set unit KB
+      }
+    }
+    puts [format {%7.1f %s index %s of table %s} \
+            [expr {$sz/$scale}] $unit $name $tbl_name]
+  }
+}
diff --git a/manifest b/manifest
index 0976eed231..e4e6ce9964 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sall\srecent\senhancements\sfrom\strunk.
-D 2017-10-31T18:15:36.138
+C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary.
+D 2017-11-01T00:10:34.144
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
-F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4
+F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de
-R 6d1ce2918085f767aa6a41a268922785
+P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895
+R f819f4cdc6b92e91586a5e2d567aa2dc
 U drh
-Z 59d272efca4b8d35d600b1005fb03cf7
+Z e7d1ee3ba646c59ecaada1f36dfcbcb9
diff --git a/manifest.uuid b/manifest.uuid
index 462cf20d71..f800dc1ed5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895
\ No newline at end of file
+e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659
\ No newline at end of file

From 1fae37319e5a104161f9aa0d27e27508ac453ffa Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 01:01:20 +0000
Subject: [PATCH 244/270] Actually perform index checks when running
 sqlite3_checker

FossilOrigin-Name: 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719
---
 ext/repair/sqlite3_checker.tcl | 113 ++++++++++++++++++++++++++++++---
 manifest                       |  12 ++--
 manifest.uuid                  |   2 +-
 3 files changed, 112 insertions(+), 15 deletions(-)

diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index fa3ffd73ce..d4aa84aa3b 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -42,6 +42,37 @@ proc tclsh {} {
   }
 }
 
+# Do an incremental integrity check of a single index
+#
+proc check_index {idxname batchsize} {
+  set i 0
+  set more 1
+  set nerr 0
+  puts -nonewline "$idxname: "
+  while {$more} {
+    set more 0
+    db eval {SELECT errmsg, current_key AS key
+               FROM incremental_index_check($idxname)
+              WHERE after_key=$key
+              LIMIT $batchsize} {
+      set more 1
+      if {$errmsg!=""} {
+        if {$nerr>0} {
+           puts -nonewline "$idxname: "
+        }
+        incr nerr
+        puts "row $i: $errmsg"
+      }
+      incr i
+    }
+  }
+  if {$nerr==0} {
+    puts "$i entries, ok"
+  } else {
+    puts "$idxname: $nerr errors out of $i entries"
+  }
+}
+
 # Print a usage message on standard error, then quit.
 #
 proc usage {} {
@@ -53,11 +84,19 @@ Do sanity checking on a live SQLite3 database file specified by the
 
 Options:
 
-   --freelist   Perform a freelist check
+   --batchsize N     Number of rows to check per transaction
 
-   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
+   --freelist        Perform a freelist check
 
-   --version    Show the version number of SQLite
+   --index NAME      Run a check of the index NAME
+
+   --summary         Print summary information about the database
+
+   --table NAME      Run a check of all indexes for table NAME
+
+   --tclsh           Run the built-in TCL interpreter (for debugging)
+
+   --version         Show the version number of SQLite
 }
   exit 1
 }
@@ -65,8 +104,14 @@ Options:
 set file_to_analyze {}
 append argv {}
 set bFreelistCheck 0
-set bSummary 1
-foreach arg $argv {
+set bSummary 0
+set zIndex {}
+set zTable {}
+set batchsize 100
+set bAll 1
+set argc [llength $argv]
+for {set i 0} {$i<$argc} {incr i} {
+  set arg [lindex $argv $i]
   if {[regexp {^-+tclsh$} $arg]} {
     tclsh
     exit 0
@@ -79,7 +124,41 @@ foreach arg $argv {
   }
   if {[regexp {^-+freelist$} $arg]} {
     set bFreelistCheck 1
-    set bSummary 0
+    set bAll 0
+    continue
+  }
+  if {[regexp {^-+summary$} $arg]} {
+    set bSummary 1
+    set bAll 0
+    continue
+  }
+  if {[regexp {^-+batchsize$} $arg]} {
+    incr i
+    if {$i>=$argc} {
+      puts stderr "missing argument on $arg"
+      exit 1
+    }
+    set batchsize [lindex $argv $i]
+    continue
+  }
+  if {[regexp {^-+index$} $arg]} {
+    incr i
+    if {$i>=$argc} {
+      puts stderr "missing argument on $arg"
+      exit 1
+    }
+    set zIndex [lindex $argv $i]
+    set bAll 0
+    continue
+  }
+  if {[regexp {^-+table$} $arg]} {
+    incr i
+    if {$i>=$argc} {
+      puts stderr "missing argument on $arg"
+      exit 1
+    }
+    set zTable [lindex $argv $i]
+    set bAll 0
     continue
   }
   if {[regexp {^-} $arg]} {
@@ -118,8 +197,8 @@ if {[catch {sqlite3 db $file_to_analyze} res]} {
   exit 1
 }
 
-if {$bFreelistCheck} {
-  puts "freelist-check:"
+if {$bFreelistCheck || $bAll} {
+  puts -nonewline "freelist-check: "
   flush stdout
   puts [db one {SELECT checkfreelist('main')}]
 }
@@ -143,3 +222,21 @@ if {$bSummary} {
             [expr {$sz/$scale}] $unit $name $tbl_name]
   }
 }
+if {$zIndex!=""} {
+  check_index $zIndex $batchsize
+}
+if {$zTable!=""} {
+  foreach idx [db eval {SELECT name FROM sqlite_master
+                         WHERE type='index' AND rootpage>0
+                           AND tbl_name=$zTable}] {
+    check_index $idx $batchsize
+  }
+}
+if {$bAll} {
+  set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main')
+                        WHERE type='index' AND rootpage>0
+                        ORDER BY nEntry}]
+  foreach idx $allidx {
+    check_index $idx $batchsize
+  }
+}
diff --git a/manifest b/manifest
index e4e6ce9964..a675da0f12 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary.
-D 2017-11-01T00:10:34.144
+C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker
+D 2017-11-01T01:01:20.360
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
-F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967
+F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895
-R f819f4cdc6b92e91586a5e2d567aa2dc
+P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659
+R 6f23f39130aeaf4ea9f27fe6626dd3eb
 U drh
-Z e7d1ee3ba646c59ecaada1f36dfcbcb9
+Z cc97ad1013bf49d4724282a689bb05ce
diff --git a/manifest.uuid b/manifest.uuid
index f800dc1ed5..3e765dc206 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659
\ No newline at end of file
+54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719
\ No newline at end of file

From 24fa4d57a72de204c782e2e48664ae2484a7c2e8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 01:05:41 +0000
Subject: [PATCH 245/270] When sqlite3_checker finds a problem, show the row
 key as part of the error message, not the row index number.

FossilOrigin-Name: 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df
---
 ext/repair/sqlite3_checker.tcl |  2 +-
 manifest                       | 12 ++++++------
 manifest.uuid                  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index d4aa84aa3b..25a5d252e6 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -61,7 +61,7 @@ proc check_index {idxname batchsize} {
            puts -nonewline "$idxname: "
         }
         incr nerr
-        puts "row $i: $errmsg"
+        puts "key($key): $errmsg"
       }
       incr i
     }
diff --git a/manifest b/manifest
index a675da0f12..222cd77113 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker
-D 2017-11-01T01:01:20.360
+C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber.
+D 2017-11-01T01:05:42.000
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
-F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e
+F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659
-R 6f23f39130aeaf4ea9f27fe6626dd3eb
+P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719
+R 84bd4abb8e94f0df6079dbff9fcf5a09
 U drh
-Z cc97ad1013bf49d4724282a689bb05ce
+Z d157fe339c4385e31979ffec5d5c510b
diff --git a/manifest.uuid b/manifest.uuid
index 3e765dc206..69455d17b6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719
\ No newline at end of file
+6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df
\ No newline at end of file

From 02c21b7e34a35755b3e90bf5fb352af46e7df187 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 13:09:02 +0000
Subject: [PATCH 246/270] Minor cleanup in checkindex.c.  Add progress displays
 when checking a single index in the top-level TCL script for sqlite3_checker.

FossilOrigin-Name: 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3
---
 ext/repair/checkindex.c        | 17 ++++++++++++-----
 ext/repair/sqlite3_checker.tcl | 25 ++++++++++++++-----------
 manifest                       | 14 +++++++-------
 manifest.uuid                  |  2 +-
 4 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index 16fb3b327e..362e199168 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -42,6 +42,7 @@ struct CidxTable {
 
 struct CidxCursor {
   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_int64 iRowid;
   sqlite3_stmt *pStmt;
 };
 
@@ -93,6 +94,10 @@ static int cidxConnect(
   int rc = SQLITE_OK;
   CidxTable *pRet;
 
+#define IIC_ERRMSG        0
+#define IIC_CURRENT_KEY   1
+#define IIC_INDEX_NAME    2
+#define IIC_AFTER_KEY     3
   rc = sqlite3_declare_vtab(db,
       "CREATE TABLE xyz("
       " errmsg TEXT, current_key TEXT,"
@@ -130,10 +135,10 @@ static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){
     if( p->usable==0 ) continue;
     if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
 
-    if( p->iColumn==2 ){
+    if( p->iColumn==IIC_INDEX_NAME ){
       iIdxName = i;
     }
-    if( p->iColumn==3 ){
+    if( p->iColumn==IIC_AFTER_KEY ){
       iAfterKey = i;
     }
   }
@@ -193,6 +198,7 @@ static int cidxNext(sqlite3_vtab_cursor *pCursor){
       cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db));
     }
   }else{
+    pCsr->iRowid++;
     rc = SQLITE_OK;
   }
   return rc;
@@ -737,6 +743,7 @@ static int cidxFilter(
     assert( rc==SQLITE_OK );
     rc = cidxNext(pCursor);
   }
+  pCsr->iRowid = 1;
   return rc;
 }
 
@@ -749,8 +756,8 @@ static int cidxColumn(
   int iCol
 ){
   CidxCursor *pCsr = (CidxCursor*)pCursor;
-  assert( iCol==0 || iCol==1 );
-  if( iCol==0 ){
+  assert( iCol>=IIC_ERRMSG && iCol<=IIC_AFTER_KEY );
+  if( iCol==IIC_ERRMSG ){
     const char *zVal = 0;
     if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){
       if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){
@@ -760,7 +767,7 @@ static int cidxColumn(
       zVal = "row missing";
     }
     sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC);
-  }else{
+  }else if( iCol==IIC_CURRENT_KEY ){
     sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1));
   }
   return SQLITE_OK;
diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index 25a5d252e6..a29d0332a6 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -48,7 +48,11 @@ proc check_index {idxname batchsize} {
   set i 0
   set more 1
   set nerr 0
-  puts -nonewline "$idxname: "
+  set pct 00.0
+  set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main')
+                    WHERE name=$idxname}]
+  puts -nonewline "$idxname: $i of $max rows ($pct%)\r"
+  flush stdout
   while {$more} {
     set more 0
     db eval {SELECT errmsg, current_key AS key
@@ -57,20 +61,19 @@ proc check_index {idxname batchsize} {
               LIMIT $batchsize} {
       set more 1
       if {$errmsg!=""} {
-        if {$nerr>0} {
-           puts -nonewline "$idxname: "
-        }
         incr nerr
-        puts "key($key): $errmsg"
+        puts "$idxname: key($key): $errmsg"
       }
       incr i
     }
+    set x [format {%.1f} [expr {($i*100.0)/$max}]]
+    if {$x!=$pct} {
+      puts -nonewline "$idxname: $i of $max rows ($pct%)\r"
+      flush stdout
+      set pct $x
+    }
   }
-  if {$nerr==0} {
-    puts "$i entries, ok"
-  } else {
-    puts "$idxname: $nerr errors out of $i entries"
-  }
+  puts "$idxname: $nerr errors out of $i entries"
 }
 
 # Print a usage message on standard error, then quit.
@@ -107,7 +110,7 @@ set bFreelistCheck 0
 set bSummary 0
 set zIndex {}
 set zTable {}
-set batchsize 100
+set batchsize 1000
 set bAll 1
 set argc [llength $argv]
 for {set i 0} {$i<$argc} {incr i} {
diff --git a/manifest b/manifest
index 222cd77113..bf8e3e2133 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber.
-D 2017-11-01T01:05:42.000
+C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker.
+D 2017-11-01T13:09:02.677
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -328,9 +328,9 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206
+F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9
 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
-F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7
+F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719
-R 84bd4abb8e94f0df6079dbff9fcf5a09
+P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df
+R 9a8d975a41c4365cfebb7312db115daa
 U drh
-Z d157fe339c4385e31979ffec5d5c510b
+Z d02ff4965492ed83b018c8e578de7329
diff --git a/manifest.uuid b/manifest.uuid
index 69455d17b6..86f32e4c10 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df
\ No newline at end of file
+3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3
\ No newline at end of file

From 37ab9523fa97daea32e40e5ed111acf1f91dc78b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 18:05:32 +0000
Subject: [PATCH 247/270] Move the test scripts for checkfreelist and
 checkindex over into the ext/repair/test directory.  Run them now using the
 sqlite3_checker utility with the --test option.  Some tests are currently
 failing due to an incomplete port.  This is an incremental check-in.

FossilOrigin-Name: 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7
---
 ext/repair/sqlite3_checker.tcl                |  8 +--
 ext/repair/test/README.md                     | 13 ++++
 .../repair/test/checkfreelist01.test          | 33 +--------
 .../repair/test/checkindex01.test             | 38 +----------
 ext/repair/test/test.tcl                      | 67 +++++++++++++++++++
 manifest                                      | 18 ++---
 manifest.uuid                                 |  2 +-
 7 files changed, 98 insertions(+), 81 deletions(-)
 create mode 100644 ext/repair/test/README.md
 rename test/checkfreelist.test => ext/repair/test/checkfreelist01.test (75%)
 rename test/checkindex.test => ext/repair/test/checkindex01.test (90%)
 create mode 100644 ext/repair/test/test.tcl

diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index a29d0332a6..ab8d05e600 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -1,5 +1,3 @@
-# Read and run TCL commands from standard input.  Used to implement
-# the --tclsh option.
 # This TCL script is the main driver script for the sqlite3_checker utility
 # program.
 #
@@ -10,10 +8,10 @@
 #
 # uses FILENAME in place of this script.
 #
-if {[lindex $argv 0]=="--test" && [llength $argv]>2} {
-  set file [lindex $argv 1]
+if {[lindex $argv 0]=="--test" && [llength $argv]>1} {
+  set ::argv0 [lindex $argv 1]
   set argv [lrange $argv 2 end]
-  source $file
+  source $argv0
   exit 0
 }
 
diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md
new file mode 100644
index 0000000000..8cc954adf5
--- /dev/null
+++ b/ext/repair/test/README.md
@@ -0,0 +1,13 @@
+To run these tests, first build sqlite3_checker:
+
+
+>     make sqlite3_checker
+
+
+Then run the "test.tcl" script using:
+
+
+>     ./sqlite3_checker --test $path/test.tcl
+
+
+Optionally add the full pathnames of individual *.test modules
diff --git a/test/checkfreelist.test b/ext/repair/test/checkfreelist01.test
similarity index 75%
rename from test/checkfreelist.test
rename to ext/repair/test/checkfreelist01.test
index 93e4ecc234..7c33862583 100644
--- a/test/checkfreelist.test
+++ b/ext/repair/test/checkfreelist01.test
@@ -1,40 +1,12 @@
 # 2017-10-11
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this file is testing the checkfreelist extension.
-#
 
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
 set testprefix checkfreelist
 
-ifcapable !vtab||!compound {
-  finish_test
-  return
-}
-
-if {[file exists ../checkfreelist.so]==0} {
-  finish_test
-  return
-}
-
 do_execsql_test 1.0 {
+  PRAGMA page_size=1024;
   CREATE TABLE t1(a, b);
 }
 
-db enable_load_extension 1
-do_execsql_test 1.1 {
-  SELECT load_extension('../checkfreelist.so');
-} {{}}
-
 do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok}
 do_execsql_test 1.3 {
   WITH s(i) AS (
@@ -118,6 +90,3 @@ do_execsql_test 1.11 {
     SELECT checkfreelist('main');
   ROLLBACK;
 } {{leaf count out of range (249) on trunk page 5}}
-
-finish_test
-
diff --git a/test/checkindex.test b/ext/repair/test/checkindex01.test
similarity index 90%
rename from test/checkindex.test
rename to ext/repair/test/checkindex01.test
index 22463d3ffe..36bae886d2 100644
--- a/test/checkindex.test
+++ b/ext/repair/test/checkindex01.test
@@ -1,31 +1,7 @@
 # 2017-10-11
 #
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this file is testing the checkindex extension.
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
 set testprefix checkindex
 
-ifcapable !vtab||!compound {
-  finish_test
-  return
-}
-
-if {[file exists ../checkindex.so]==0} {
-  finish_test
-  return
-}
-
 do_execsql_test 1.0 {
   CREATE TABLE t1(a, b);
   CREATE INDEX i1 ON t1(a);
@@ -36,12 +12,7 @@ do_execsql_test 1.0 {
   INSERT INTO t1 VALUES('five', 10);
 
   CREATE INDEX i2 ON t1(a DESC);
-}
-
-db enable_load_extension 1
-do_execsql_test 1.1 {
-  SELECT load_extension('../checkindex.so');
-} {{}}
+} {}
 
 proc incr_index_check {idx nStep} {
   set Q {
@@ -148,7 +119,7 @@ do_execsql_test 2.0 {
   CREATE INDEX i3 ON t2(b, c, d);
   CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC);
   CREATE INDEX i5 ON t2(d, c DESC, b);
-}
+} {}
 
 do_index_check_test 2.1 i3 {
   {} NULL,1,1,1 
@@ -205,7 +176,7 @@ do_execsql_test 3.0 {
   INSERT INTO t3 VALUES('a', 'b', NULL, 8);
   INSERT INTO t3 VALUES('a', 'b', NULL, 9);
 
-}
+} {}
 
 do_index_check_test 3.1 t3wxy {
   {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 
@@ -345,6 +316,3 @@ do_index_check_test 6.2 t6x3 {
   {} 3,2,1 
   {} 6,5,4
 }
-
-finish_test
-
diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl
new file mode 100644
index 0000000000..c073bb73c5
--- /dev/null
+++ b/ext/repair/test/test.tcl
@@ -0,0 +1,67 @@
+# Run this script using
+#
+#       sqlite3_checker --test $thisscript $testscripts
+#
+# The $testscripts argument is optional.  If omitted, all *.test files
+# in the same directory as $thisscript are run.
+#
+set NTEST 0
+set NERR  0
+
+
+# Invoke the do_test procedure to run a single test
+#
+# The $expected parameter is the expected result.  The result is the return
+# value from the last TCL command in $cmd.
+#
+# Normally, $expected must match exactly.  But if $expected is of the form
+# "/regexp/" then regular expression matching is used.  If $expected is
+# "~/regexp/" then the regular expression must NOT match.  If $expected is
+# of the form "#/value-list/" then each term in value-list must be numeric
+# and must approximately match the corresponding numeric term in $result.
+# Values must match within 10%.  Or if the $expected term is A..B then the
+# $result term must be in between A and B.
+#
+proc do_test {name cmd expected} {
+  if {[info exists ::testprefix]} {
+    set name "$::testprefix$name"
+  }
+
+  incr ::NTEST
+  puts -nonewline $name...
+  flush stdout
+
+  if {[catch {uplevel #0 "$cmd;\n"} result]} {
+    puts -nonewline $name...
+    puts "\nError: $result"
+    incr ::NERR
+  } else {
+    set ok [expr {[string compare $result $expected]==0}]
+    if {!$ok} {
+      puts "\n!  $name expected: \[$expected\]\n! $name got:      \[$result\]"
+      incr ::NERR
+    } else {
+      puts " Ok"
+    }
+  }
+  flush stdout
+}
+
+#
+#   do_execsql_test TESTNAME SQL RES
+#
+proc do_execsql_test {testname sql {result {}}} {
+  uplevel [list do_test $testname [list db eval $sql] [list {*}$result]]
+}
+
+if {[llength $argv]==0} {
+  set dir [file dirname $argv0]
+  set argv [glob -nocomplain $dir/*.test]
+}
+foreach testfile $argv {
+  file delete -force test.db
+  sqlite3 db test.db
+  source $testfile
+  catch {db close}
+}
+puts "$NERR errors out of $NTEST tests"
diff --git a/manifest b/manifest
index bf8e3e2133..ff5e3f094d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker.
-D 2017-11-01T13:09:02.677
+C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in.
+D 2017-11-01T18:05:32.697
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -330,7 +330,11 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9
 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
-F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1
+F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
+F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
+F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f
+F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7
+F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -655,8 +659,6 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe
 F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
-F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1671,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df
-R 9a8d975a41c4365cfebb7312db115daa
+P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3
+R 9a0909f5ab1dfc6f8f86ede9c43722ca
 U drh
-Z d02ff4965492ed83b018c8e578de7329
+Z 17e4966dab1f438179fdac475786b16e
diff --git a/manifest.uuid b/manifest.uuid
index 86f32e4c10..1cd31e3773 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3
\ No newline at end of file
+17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7
\ No newline at end of file

From 3680e89b1e0fe927445e7046dee037957818fe1b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 18:31:34 +0000
Subject: [PATCH 248/270] Add the "sqlite3_imposter DB SCHEMA ROOT SQL" command
 to sqlite3_checker. Use it to fix the checkindex01.test module.  There are
 still errors reported by the checkfreelist01.test module.

FossilOrigin-Name: 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2
---
 ext/repair/sqlite3_checker.c.in   | 52 +++++++++++++++++++++++++++++++
 ext/repair/test/checkindex01.test | 29 +++++++----------
 manifest                          | 14 ++++-----
 manifest.uuid                     |  2 +-
 4 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in
index 6045c36a9d..96b15f2713 100644
--- a/ext/repair/sqlite3_checker.c.in
+++ b/ext/repair/sqlite3_checker.c.in
@@ -19,9 +19,61 @@ INCLUDE $ROOT/ext/misc/btreeinfo.c
 INCLUDE $ROOT/ext/repair/checkindex.c
 INCLUDE $ROOT/ext/repair/checkfreelist.c
 
+/*
+** Decode a pointer to an sqlite3 object.
+*/
+int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){
+  struct SqliteDb *p;
+  Tcl_CmdInfo cmdInfo;
+  if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){
+    p = (struct SqliteDb*)cmdInfo.objClientData;
+    *ppDb = p->db;
+    return TCL_OK;
+  }else{
+    *ppDb = 0;
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
+/*
+**   sqlite3_imposter db main rootpage {CREATE TABLE...}  ;# setup an imposter
+**   sqlite3_imposter db main                             ;# rm all imposters
+*/
+static int sqlite3_imposter(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  const char *zSchema;
+  int iRoot;
+  const char *zSql;
+
+  if( objc!=3 && objc!=5 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zSchema = Tcl_GetString(objv[2]);
+  if( objc==3 ){
+    sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1);
+  }else{
+    if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR;
+    zSql = Tcl_GetString(objv[4]);
+    sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot);
+    sqlite3_exec(db, zSql, 0, 0, 0);
+    sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0);
+  }
+  return TCL_OK;
+}
+
 #include <stdio.h>
 
 const char *sqlite3_checker_init_proc(Tcl_Interp *interp){
+  Tcl_CreateObjCommand(interp, "sqlite3_imposter", 
+                       (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0);
   sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init);
   sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init);
   sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init);
diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test
index 36bae886d2..744a21c69e 100644
--- a/ext/repair/test/checkindex01.test
+++ b/ext/repair/test/checkindex01.test
@@ -71,16 +71,12 @@ do_index_check_test 1.4 i2 {
 
 do_test 1.5 {
   set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }]
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
-  db eval {CREATE TABLE xt1(a, b)}
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
-
-  execsql {
+  sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)}
+  db eval {
     UPDATE xt1 SET a='six' WHERE rowid=3;
     DELETE FROM xt1 WHERE rowid = 5;
   }
-
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+  sqlite3_imposter db main
 } {}
 
 do_index_check_test 1.6 i1 {
@@ -211,16 +207,14 @@ do_index_check_test 4.1 t4cc {
 
 do_test 4.2 {
   set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }]
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
-  db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)}
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
+  sqlite3_imposter db main $tblroot \
+     {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)}
 
-  execsql {
+  db eval {
     UPDATE xt4 SET c1='hello' WHERE rowid=2;
     DELETE FROM xt4 WHERE rowid = 3;
   }
-
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+  sqlite3_imposter db main
 } {}
 
 do_index_check_test 4.3 t4cc {
@@ -263,14 +257,13 @@ do_index_check_test 5.1.3 sqlite_autoindex_t5_1 {
 
 do_test 5.2 {
   set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }]
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
-  db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);}
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
-  execsql {
+  sqlite3_imposter db main $tblroot \
+      {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);}
+  db eval {
     UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1;
     DELETE FROM xt5 WHERE rowid = 4;
   }
-  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+  sqlite3_imposter db main
 } {}
 
 do_index_check_test 5.3.1 t5x {
diff --git a/manifest b/manifest
index ff5e3f094d..1e2056b04c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in.
-D 2017-11-01T18:05:32.697
+C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule.
+D 2017-11-01T18:31:34.119
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -329,11 +329,11 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9
-F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510
+F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7
 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
 F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f
-F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7
+F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3
-R 9a0909f5ab1dfc6f8f86ede9c43722ca
+P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7
+R 789d2ddc9f5813af408c7050793744d3
 U drh
-Z 17e4966dab1f438179fdac475786b16e
+Z 00e34b3226a7a4c513950643ceed7a0f
diff --git a/manifest.uuid b/manifest.uuid
index 1cd31e3773..6e77603c23 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7
\ No newline at end of file
+0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2
\ No newline at end of file

From 9b37b366c2624de4a15e216b34c3d4620bfa42da Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 19:22:08 +0000
Subject: [PATCH 249/270] Adjust page numbers in the
 ext/repair/test/checkfreelist01.test module due to the fact that the
 pending-byte page is no longer shifted down to a low-numbered page but is in
 its rightful place.

FossilOrigin-Name: c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78
---
 ext/repair/test/checkfreelist01.test | 14 +++++++-------
 manifest                             | 12 ++++++------
 manifest.uuid                        |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test
index 7c33862583..7e2dd51c37 100644
--- a/ext/repair/test/checkfreelist01.test
+++ b/ext/repair/test/checkfreelist01.test
@@ -27,8 +27,8 @@ do_execsql_test 1.5 {
   )
   SELECT i FROM freelist_trunk WHERE i!=1;
 } {
-  10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235
-  4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5
+  10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234
+  4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5
 }
 
 do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok}
@@ -50,7 +50,7 @@ do_execsql_test 1.7 {
   BEGIN;
     UPDATE sqlite_dbpage 
       SET data = set_int(data, 1, get_int(data, 1)-1) 
-      WHERE pgno=4861;
+      WHERE pgno=4860;
     SELECT checkfreelist('main');
   ROLLBACK;
 } {{free-list count mismatch: actual=6725 header=6726}}
@@ -59,19 +59,19 @@ do_execsql_test 1.8 {
   BEGIN;
     UPDATE sqlite_dbpage 
       SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1)
-      WHERE pgno=4861;
+      WHERE pgno=4860;
     SELECT checkfreelist('main');
   ROLLBACK;
-} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}}
+} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}}
 
 do_execsql_test 1.9 {
   BEGIN;
     UPDATE sqlite_dbpage 
       SET data = set_int(data, 5, 0)
-      WHERE pgno=4861;
+      WHERE pgno=4860;
     SELECT checkfreelist('main');
   ROLLBACK;
-} {{leaf page 0 is out of range (child 3 of trunk page 4861)}}
+} {{leaf page 0 is out of range (child 3 of trunk page 4860)}}
 
 do_execsql_test 1.10 {
   BEGIN;
diff --git a/manifest b/manifest
index 1e2056b04c..4d5258d691 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule.
-D 2017-11-01T18:31:34.119
+C Adjust\spage\snumbers\sin\sthe\sext/repair/test/checkfreelist01.test\smodule\sdue\sto\nthe\sfact\sthat\sthe\spending-byte\spage\sis\sno\slonger\sshifted\sdown\sto\sa\slow-numbered\npage\sbut\sis\sin\sits\srightful\splace.
+D 2017-11-01T19:22:08.991
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -332,7 +332,7 @@ F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89
 F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7
 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
-F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f
+F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69
 F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7
-R 789d2ddc9f5813af408c7050793744d3
+P 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2
+R e97da7e6f34a0c98621feff75b5fc1d7
 U drh
-Z 00e34b3226a7a4c513950643ceed7a0f
+Z 33cbf67c611e121ff7dcf902c08c95ea
diff --git a/manifest.uuid b/manifest.uuid
index 6e77603c23..6eea893785 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2
\ No newline at end of file
+c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78
\ No newline at end of file

From 7dcde38f59eb0b5d858597147f7f8725d2672764 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 19:50:11 +0000
Subject: [PATCH 250/270] Make sure the JSON1 extension is available to
 sqlite3_checker.exe

FossilOrigin-Name: a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de
---
 ext/repair/sqlite3_checker.c.in |  1 +
 manifest                        | 13 ++++++-------
 manifest.uuid                   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in
index 96b15f2713..76e9708787 100644
--- a/ext/repair/sqlite3_checker.c.in
+++ b/ext/repair/sqlite3_checker.c.in
@@ -4,6 +4,7 @@
 */
 #define TCLSH_INIT_PROC sqlite3_checker_init_proc
 #define SQLITE_ENABLE_DBPAGE_VTAB 1
+#define SQLITE_ENABLE_JSON1 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0
 #undef SQLITE_ENABLE_COLUMN_METADATA
diff --git a/manifest b/manifest
index 86482e8b0c..47e05b0082 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\scheckindex.c\sextension\sand\sthe\ssqlite3_checker\sutility\sprogram\sused\nfor\sdoing\slive\svalidation\sof\slarge\sdatabases.
-D 2017-11-01T19:44:19.216
+C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe
+D 2017-11-01T19:50:11.569
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9
-F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7
+F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f
 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
 F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69
@@ -1673,8 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78
-R e97da7e6f34a0c98621feff75b5fc1d7
-T +closed c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78
+P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6
+R f83c5c1798f6f791a77e395a421e7535
 U drh
-Z 29d052b9d1f9214dbd3b25e5b7544e56
+Z b827e3b3bff13979721cde0c77e4ccc0
diff --git a/manifest.uuid b/manifest.uuid
index b0b6dde090..05400f1a8f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6
\ No newline at end of file
+a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de
\ No newline at end of file

From a48a290bcc6f7f81d0378aa5fc4ad51ee6e182cc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 1 Nov 2017 19:58:25 +0000
Subject: [PATCH 251/270] Fix harmless compiler warnings in sqlite3_checker.exe

FossilOrigin-Name: 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0
---
 ext/repair/checkfreelist.c |  2 +-
 ext/repair/checkindex.c    |  4 ++--
 manifest                   | 16 ++++++++--------
 manifest.uuid              |  2 +-
 src/dbpage.c               |  2 +-
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c
index cd2801e040..990be4afa7 100644
--- a/ext/repair/checkfreelist.c
+++ b/ext/repair/checkfreelist.c
@@ -166,7 +166,7 @@ static int checkFreelist(
     u32 i;
     u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0);
     const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1);
-    int nData = sqlite3_column_bytes(pTrunk, 1);
+    u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1);
     u32 iNext = get4byte(&aData[0]);
     u32 nLeaf = get4byte(&aData[4]);
 
diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index 362e199168..fa8c713f7c 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -263,7 +263,7 @@ static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){
 char *cidxStrdup(int *pRc, const char *zStr){
   char *zRet = 0;
   if( *pRc==SQLITE_OK ){
-    int n = strlen(zStr);
+    int n = (int)strlen(zStr);
     zRet = cidxMalloc(pRc, n+1);
     if( zRet ) memcpy(zRet, zStr, n+1);
   }
@@ -493,7 +493,7 @@ static int cidxDecodeAfter(
 ){
   char **azAfter;
   int rc = SQLITE_OK;
-  int nAfterKey = strlen(zAfterKey);
+  int nAfterKey = (int)strlen(zAfterKey);
 
   azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1);
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 47e05b0082..9e0fd087b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe
-D 2017-11-01T19:50:11.569
+C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe
+D 2017-11-01T19:58:25.196
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -327,8 +327,8 @@ F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078
 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
-F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9
+F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd
+F ext/repair/checkindex.c 73f26fc1e2e17d68ede5db5b0aaf4869f2d6182f45f3d3624befc503c0f04a70
 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f
 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
@@ -420,7 +420,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
-F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2
+F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6
-R f83c5c1798f6f791a77e395a421e7535
+P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de
+R 7ee45d8c7d2f00dd882d0691ca6a1bfb
 U drh
-Z b827e3b3bff13979721cde0c77e4ccc0
+Z 0f28cc7179e20c43d298dd1935e3a35c
diff --git a/manifest.uuid b/manifest.uuid
index 05400f1a8f..7f5fcadcf3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de
\ No newline at end of file
+491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
index f287e72229..75e4c42e67 100644
--- a/src/dbpage.c
+++ b/src/dbpage.c
@@ -329,7 +329,7 @@ static int dbpageUpdate(
     goto update_fail;
   }
   pBt = pTab->db->aDb[iDb].pBt;
-  if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){
+  if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){
     zErr = "bad page number";
     goto update_fail;
   }

From 53be36b026469d665ed238dc4f65f37161862c7b Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 3 Nov 2017 06:45:37 +0000
Subject: [PATCH 252/270] Fix harmless compiler warnings.

FossilOrigin-Name: d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/whereexpr.c |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 9e0fd087b8..c6be978982 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe
-D 2017-11-01T19:58:25.196
+C Fix\sharmless\scompiler\swarnings.
+D 2017-11-03T06:45:37.503
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -556,7 +556,7 @@ F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2
-F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4
+F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de
-R 7ee45d8c7d2f00dd882d0691ca6a1bfb
-U drh
-Z 0f28cc7179e20c43d298dd1935e3a35c
+P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0
+R 2898c172e35d6c4a6ab6316ba23aa96d
+U mistachkin
+Z 0198e72b5c24cd8ac4cd7a2d9de73913
diff --git a/manifest.uuid b/manifest.uuid
index 7f5fcadcf3..d96908a8ac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0
\ No newline at end of file
+d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
\ No newline at end of file
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 9f83a84534..58f1908cf8 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -983,7 +983,7 @@ static void exprAnalyze(
   int op;                          /* Top-level operator.  pExpr->op */
   Parse *pParse = pWInfo->pParse;  /* Parsing context */
   sqlite3 *db = pParse->db;        /* Database connection */
-  unsigned char eOp2;              /* op2 value for LIKE/REGEXP/GLOB */
+  unsigned char eOp2 = 0;          /* op2 value for LIKE/REGEXP/GLOB */
   int nLeft;                       /* Number of elements on left side vector */
 
   if( db->mallocFailed ){
@@ -1227,7 +1227,7 @@ static void exprAnalyze(
   ** to do anything with MATCH functions.
   */
   if( pWC->op==TK_AND ){
-    Expr *pRight, *pLeft;
+    Expr *pRight = 0, *pLeft = 0;
     int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight);
     while( res-- > 0 ){
       int idxNew;

From 537e70289ff4168476252458965ca0bccf70559b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 3 Nov 2017 08:46:48 +0000
Subject: [PATCH 253/270] The extensions functions in spellfix are all
 deterministic.

FossilOrigin-Name: 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
---
 ext/misc/spellfix.c | 24 +++++++++++++++---------
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 1ac1712f4e..4f17b88e1d 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -1122,15 +1122,17 @@ static int editDist3Install(sqlite3 *db){
   if( pConfig==0 ) return SQLITE_NOMEM;
   memset(pConfig, 0, sizeof(*pConfig));
   rc = sqlite3_create_function_v2(db, "editdist3",
-              2, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0);
+              2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+              editDist3SqlFunc, 0, 0, 0);
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function_v2(db, "editdist3",
-                3, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0);
+                3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+                editDist3SqlFunc, 0, 0, 0);
   }
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function_v2(db, "editdist3",
-                1, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0,
-                editDist3ConfigDelete);
+                1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+                editDist3SqlFunc, 0, 0, editDist3ConfigDelete);
   }else{
     sqlite3_free(pConfig);
   }
@@ -2895,18 +2897,22 @@ static sqlite3_module spellfix1Module = {
 static int spellfix1Register(sqlite3 *db){
   int rc = SQLITE_OK;
   int i;
-  rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0,
-                                  transliterateSqlFunc, 0, 0);
+  rc = sqlite3_create_function(db, "spellfix1_translit", 1,
+                               SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
+                                transliterateSqlFunc, 0, 0);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "spellfix1_editdist", 2,
+                                 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
                                   editdistSqlFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "spellfix1_phonehash", 1,
+                                 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
                                   phoneticHashSqlFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1,
+                                  SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
                                   scriptCodeSqlFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index c6be978982..a447c36649 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings.
-D 2017-11-03T06:45:37.503
+C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic.
+D 2017-11-03T08:46:48.968
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -282,7 +282,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d
+F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0
-R 2898c172e35d6c4a6ab6316ba23aa96d
-U mistachkin
-Z 0198e72b5c24cd8ac4cd7a2d9de73913
+P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
+R b7838526febd368042ba32a4576c580c
+U drh
+Z 18ebbc1917110c35a4b2606f5ce6f533
diff --git a/manifest.uuid b/manifest.uuid
index d96908a8ac..5035b0e5a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
\ No newline at end of file
+29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
\ No newline at end of file

From a3bc84255a1145a0f04359717fe7e35f70f88c41 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 4 Nov 2017 08:53:37 +0000
Subject: [PATCH 254/270] Fix an LSM problem with read-only connections and
 compression (not compression-factory) callbacks.

FossilOrigin-Name: 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e
---
 ext/lsm1/lsm_shared.c | 12 +++++-------
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c
index c4bb9d6a23..2fdacf1eca 100644
--- a/ext/lsm1/lsm_shared.c
+++ b/ext/lsm1/lsm_shared.c
@@ -520,13 +520,11 @@ int lsmDbDatabaseConnect(
   ** recovery as necessary. Or, if this is a read-only database handle,
   ** defer attempting to connect to the system until a read-transaction
   ** is opened.  */
-  if( pDb->bReadonly==0 ){
-    if( rc==LSM_OK ){
-      rc = lsmFsConfigure(pDb);
-    }
-    if( rc==LSM_OK ){
-      rc = doDbConnect(pDb);
-    }
+  if( rc==LSM_OK ){
+    rc = lsmFsConfigure(pDb);
+  }
+  if( rc==LSM_OK && pDb->bReadonly==0 ){
+    rc = doDbConnect(pDb);
   }
 
   return rc;
diff --git a/manifest b/manifest
index a447c36649..9a1f7ef8fa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic.
-D 2017-11-03T08:46:48.968
+C Fix\san\sLSM\sproblem\swith\sread-only\sconnections\sand\scompression\s(not\ncompression-factory)\scallbacks.
+D 2017-11-04T08:53:37.599
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -245,7 +245,7 @@ F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709
 F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
-F ext/lsm1/lsm_shared.c 1a76b7a5e89a003c24d58f1fb295c4203b48ef6acba9a194ac6003ade09fcd47
+F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
 F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
-R b7838526febd368042ba32a4576c580c
-U drh
-Z 18ebbc1917110c35a4b2606f5ce6f533
+P 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
+R 50deaa5f352753bc15393c76a6fab0d8
+U dan
+Z 685560e0f9310b127a265df0e79cb145
diff --git a/manifest.uuid b/manifest.uuid
index 5035b0e5a7..e247d77f4b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
\ No newline at end of file
+51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e
\ No newline at end of file

From 12f84e5ee55145c7427d071ed5afb2188e5c4ef3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 6 Nov 2017 09:34:45 +0000
Subject: [PATCH 255/270] Fix a harmless compiler warning from Xcode 9.1.

FossilOrigin-Name: 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/util.c    |  6 +++---
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 9a1f7ef8fa..f1f80d7a12 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sLSM\sproblem\swith\sread-only\sconnections\sand\scompression\s(not\ncompression-factory)\scallbacks.
-D 2017-11-04T08:53:37.599
+C Fix\sa\sharmless\scompiler\swarning\sfrom\sXcode\s9.1.
+D 2017-11-06T09:34:45.112
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -537,7 +537,7 @@ F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afe
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
 F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159
+F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
-R 50deaa5f352753bc15393c76a6fab0d8
-U dan
-Z 685560e0f9310b127a265df0e79cb145
+P 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e
+R ca8ae1ded89ca445023d13a31194e801
+U drh
+Z c36ef70e8d55fddaef51df49c2bfea5f
diff --git a/manifest.uuid b/manifest.uuid
index e247d77f4b..0cd36e32cb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e
\ No newline at end of file
+66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index fbe3714b77..a4dbe8fdaf 100644
--- a/src/util.c
+++ b/src/util.c
@@ -387,12 +387,12 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   /* copy max significant digits to significand */
   while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
     s = s*10 + (*z - '0');
-    z+=incr, nDigits++;
+    z+=incr; nDigits++;
   }
 
   /* skip non-significant significand digits
   ** (increase exponent by d to shift decimal left) */
-  while( z<zEnd && sqlite3Isdigit(*z) ) z+=incr, nDigits++, d++;
+  while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; nDigits++; d++; }
   if( z>=zEnd ) goto do_atof_calc;
 
   /* if decimal point is present */
@@ -405,7 +405,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
         s = s*10 + (*z - '0');
         d--;
       }
-      z+=incr, nDigits++;
+      z+=incr; nDigits++;
     }
   }
   if( z>=zEnd ) goto do_atof_calc;

From c0f69fcec77830092331cc4bbc0b6b36d17a5e7a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 6 Nov 2017 10:04:45 +0000
Subject: [PATCH 256/270] Allow "BEGIN CONCURRENT" transactions to modify the
 temp schema.

FossilOrigin-Name: 0fb6d91cea347384fc081ce4c79582b365801dd4f56f5bf2ed40922bbfca0344
---
 manifest             | 15 +++++++--------
 manifest.uuid        |  2 +-
 src/vdbe.c           |  4 ++--
 test/concurrent.test | 36 ++++++++++++++++++++++++++++++++----
 4 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index 861009c901..6930c352bb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cherrypick\s[ec37ad6d08]\sinto\sthis\sbranch.\sWith\sthis\spatch,\sif\nSQLITE_SHARED_MAPPING\sis\sdefined\sat\sbuild-time\sSQLite\swill\suse\sa\ssingle\smemory\nmapping\sfor\smultiple\sconnections\sto\sthe\ssame\sdatabase\sfile\swithin\sa\ssingle\nprocess.
-D 2017-09-22T11:09:09.656
+C Allow\s"BEGIN\sCONCURRENT"\stransactions\sto\smodify\sthe\stemp\sschema.
+D 2017-11-06T10:04:45.041
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -530,7 +530,7 @@ F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 5168013cfd937a695d23cce8c67cb07a3dda242d4cb812530ba1148b88e0f159
 F src/vacuum.c 07eec96f821c7dcbdca2fadffc6e38ea2c24bf409fcb15fe9fb3ac444d632dfe
-F src/vdbe.c 96467c73f53c3cb2b3ad7a73c69d59fb43966d772a7fc70114dde14cf3876dd0
+F src/vdbe.c 8b7313c7012c627a0f6579b0c1d3a42f48ffd187e4beb6a0face21092d728ac0
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -668,7 +668,7 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test c47639d26cbeba6977457e5ef2c2c55c5b6c889478dd7eb0ed858ba894e7fa93
 F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43
-F test/concurrent.test a801cd60c370f0ed851657c9576b102f9ab1dd846c6a88e6ae45939a8deeda7c
+F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e60dbde5
 F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8
 F test/concurrent3.test f4af1cf1220908c6dd5694923621c19e999b78cd997e2646285f08a52bcb4170
 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44
@@ -1666,8 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 307b802e8627c93a51e4c54851a4fab33db5061bb80e3d327ce53b127d6d511b
-Q +ec37ad6d08362f4c9faad9b629c0fa23f5864ff6ad7f4cbed93a25d5f7b815d8
-R a7069c27bdffb405c8574ad991612563
+P c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264
+R 202ef8632fda2b1b9ef7af4f1e374f95
 U dan
-Z 9a1235c2d010c6dd3774c97b07deb6f2
+Z d758dc3efdc3027768a35f11fc7171bf
diff --git a/manifest.uuid b/manifest.uuid
index da8f0bb88e..e36be45a6d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c7a5880d6d898299b4c9414b7702cfa450aa5f7bf4ec8f417b94d2a7b6558264
\ No newline at end of file
+0fb6d91cea347384fc081ce4c79582b365801dd4f56f5bf2ed40922bbfca0344
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 09c0cd32b4..4292fce260 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3306,7 +3306,7 @@ case OP_SetCookie: {
   rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3);
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
-    assert( db->bConcurrent==0 );
+    assert( pOp->p1==1 || db->bConcurrent==0 );
     pDb->pSchema->schema_cookie = pOp->p3;
     db->mDbFlags |= DBFLAG_SchemaChange;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
@@ -6509,7 +6509,7 @@ case OP_Expire: {
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
 #ifndef SQLITE_OMIT_CONCURRENT
-  if( isWriteLock && db->bConcurrent && pOp->p2==1 ){
+  if( isWriteLock && db->bConcurrent && pOp->p2==1 && pOp->p1!=1 ){
     rc = SQLITE_ERROR;
     sqlite3VdbeError(p, 
         "cannot modify database schema within CONCURRENT transaction");
diff --git a/test/concurrent.test b/test/concurrent.test
index b8f8d7e0d6..b94071a8f1 100644
--- a/test/concurrent.test
+++ b/test/concurrent.test
@@ -112,19 +112,47 @@ foreach {tn sql} {
   2 { DROP TABLE t1 }
   3 { CREATE INDEX i1 ON t1(a) }
   4 { CREATE VIEW v1 AS SELECT * FROM t1 }
-  5 { CREATE TEMP TABLE xx(a, b) }
 } {
-  do_catchsql_test 1.7.$tn.1 "
+  do_catchsql_test 1.7.0.$tn.1 "
     BEGIN CONCURRENT;
     $sql
   " {1 {cannot modify database schema within CONCURRENT transaction}}
 
-  do_execsql_test 1.7.$tn.2 {
+  do_execsql_test 1.7.0.$tn.2 {
     SELECT sql FROM sqlite_master;
     SELECT sql FROM sqlite_temp_master;
   } {{CREATE TABLE t1(a, b)}}
 
-  do_execsql_test 1.7.$tn.3 COMMIT
+  do_execsql_test 1.7.0.$tn.3 COMMIT
+}
+
+# Except the temp db schema.
+foreach {tn sql} {
+  1 { CREATE TEMP TABLE xx(a, b) }
+  2 { DROP TABLE xx }
+  3 { CREATE TEMP TABLE yy(a, b) }
+  4 { CREATE VIEW temp.v1 AS SELECT * FROM t1 }
+  5 { CREATE INDEX yyi1 ON yy(a); }
+  6 { CREATE TABLE temp.zz(a, b) }
+} {
+  do_catchsql_test 1.7.1.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {0 {}}
+
+  do_execsql_test 1.7.1.$tn.2 COMMIT
+}
+
+
+do_execsql_test 1.7.1.x {
+  SELECT sql FROM sqlite_master;
+  SELECT sql FROM sqlite_temp_master;
+} {
+  {CREATE TABLE t1(a, b)}
+  {CREATE TABLE yy(a, b)} 
+  {CREATE VIEW v1 AS SELECT * FROM t1} 
+  {CREATE INDEX yyi1 ON yy(a)} 
+  {CREATE TABLE zz(a, b)}
 }
 
 #-------------------------------------------------------------------------

From 7031b12c570689fef665eb874667620f2af382ab Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 7 Nov 2017 16:23:24 +0000
Subject: [PATCH 257/270] Enhance the checkindex.c virtual table so that it
 will output the index_name and after_key parameters.  Also add a new
 diagnostic output column named scanner_sql which shows the SQL statement used
 to implement the current index scan.

FossilOrigin-Name: 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87
---
 ext/repair/checkindex.c           | 238 +++++++++++++++++++-----------
 ext/repair/test/checkindex01.test |  24 ++-
 manifest                          |  14 +-
 manifest.uuid                     |   2 +-
 4 files changed, 175 insertions(+), 103 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index fa8c713f7c..46ecc12fc4 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -14,16 +14,18 @@
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 
+/*
+** Stuff that is available inside the amalgamation, but which we need to
+** declare ourselves if this module is compiled separately.
+*/
 #ifndef SQLITE_AMALGAMATION
 # include <string.h>
 # include <stdio.h>
 # include <stdlib.h>
 # include <assert.h>
-# define ALWAYS(X)  1
-# define NEVER(X)   0
-  typedef unsigned char u8;
-  typedef unsigned short u16;
-  typedef unsigned int u32;
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
 #define get4byte(x) (        \
     ((u32)((x)[0])<<24) +    \
     ((u32)((x)[1])<<16) +    \
@@ -42,8 +44,10 @@ struct CidxTable {
 
 struct CidxCursor {
   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
-  sqlite3_int64 iRowid;
-  sqlite3_stmt *pStmt;
+  sqlite3_int64 iRowid;           /* Row number of the output */
+  char *zIdxName;                 /* Copy of the index_name parameter */
+  char *zAfterKey;                /* Copy of the after_key parameter */
+  sqlite3_stmt *pStmt;            /* SQL statement that generates the output */
 };
 
 typedef struct CidxColumn CidxColumn;
@@ -82,7 +86,7 @@ static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){
 }
 
 /*
-** Connect to then incremental_index_check virtual table.
+** Connect to the incremental_index_check virtual table.
 */
 static int cidxConnect(
   sqlite3 *db,
@@ -98,10 +102,14 @@ static int cidxConnect(
 #define IIC_CURRENT_KEY   1
 #define IIC_INDEX_NAME    2
 #define IIC_AFTER_KEY     3
+#define IIC_SCANNER_SQL   4
   rc = sqlite3_declare_vtab(db,
       "CREATE TABLE xyz("
-      " errmsg TEXT, current_key TEXT,"
-      " index_name HIDDEN, after_key HIDDEN"
+      " errmsg TEXT,"            /* Error message or NULL if everything is ok */
+      " current_key TEXT,"       /* SQLite quote() text of key values */
+      " index_name HIDDEN,"      /* IN: name of the index being scanned */
+      " after_key HIDDEN,"       /* IN: Start scanning after this key */
+      " scanner_sql HIDDEN"      /* debuggingn info: SQL used for scanner */
       ")"
   );
   pRet = cidxMalloc(&rc, sizeof(CidxTable));
@@ -123,7 +131,15 @@ static int cidxDisconnect(sqlite3_vtab *pVtab){
 }
 
 /*
-** xBestIndex method.
+** idxNum and idxStr are not used.  There are only three possible plans,
+** which are all distinguished by the number of parameters.
+**
+**   No parameters:         A degenerate plan.  The result is zero rows.
+**   1 Parameter:           Scan all of the index starting with first entry
+**   2 parameters:          Scan the index starting after the "after_key".    
+**
+** Provide successively smaller costs for each of these plans to encourage
+** the query planner to select the one with the most parameters.
 */
 static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){
   int iIdxName = -1;
@@ -179,7 +195,8 @@ static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
 static int cidxClose(sqlite3_vtab_cursor *pCursor){
   CidxCursor *pCsr = (CidxCursor*)pCursor;
   sqlite3_finalize(pCsr->pStmt);
-  pCsr->pStmt = 0;
+  sqlite3_free(pCsr->zIdxName);
+  sqlite3_free(pCsr->zAfterKey);
   sqlite3_free(pCsr);
   return SQLITE_OK;
 }
@@ -650,6 +667,83 @@ static char *cidxColumnList(
   return zRet;
 }
 
+/*
+** Generate SQL (in memory obtained from sqlite3_malloc()) that will
+** continue the index scan for zIdxName starting after zAfterKey.
+*/
+int cidxGenerateScanSql(
+  CidxCursor *pCsr,           /* The cursor which needs the new statement */
+  const char *zIdxName,       /* index to be scanned */
+  const char *zAfterKey,      /* start after this key, if not NULL */
+  char **pzSqlOut             /* OUT: Write the generated SQL here */
+){
+  int rc;
+  char *zTab = 0;
+  char *zCurrentKey = 0;
+  char *zOrderBy = 0;
+  char *zSubWhere = 0;
+  char *zSubExpr = 0;
+  char *zSrcList = 0;
+  char **azAfter = 0;
+  CidxIndex *pIdx = 0;
+
+  *pzSqlOut = 0;
+  rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab);
+
+  zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY);
+  zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
+  zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
+  zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
+  zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL);
+
+  if( rc==SQLITE_OK && zAfterKey ){
+    rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
+  }
+
+  if( rc || zAfterKey==0 ){
+    *pzSqlOut = cidxMprintf(&rc,
+        "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
+        "FROM (SELECT %s FROM %Q ORDER BY %s) AS i",
+        zSubExpr, zTab, zSubWhere, zCurrentKey, 
+        zSrcList, zTab, zOrderBy
+    );
+  }else{
+    const char *zSep = "";
+    char *zSql;
+    int i;
+
+    zSql = cidxMprintf(&rc, 
+        "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+        zSubExpr, zTab, zSubWhere, zCurrentKey
+    );
+    for(i=pIdx->nCol-1; i>=0; i--){
+      int j;
+      if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
+      for(j=0; j<2; j++){
+        char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
+        zSql = cidxMprintf(&rc, "%z"
+            "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
+            zSql, zSep, zSrcList, zTab, zWhere, zOrderBy
+        );
+        zSep = " UNION ALL ";
+        if( pIdx->aCol[i].bDesc==0 ) break;
+      }
+    }
+    *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql);
+  }
+
+  sqlite3_free(zTab);
+  sqlite3_free(zCurrentKey);
+  sqlite3_free(zOrderBy);
+  sqlite3_free(zSubWhere);
+  sqlite3_free(zSubExpr);
+  sqlite3_free(zSrcList);
+  cidxFreeIndex(pIdx);
+  sqlite3_free(azAfter);
+  return rc;
+}
+
+
 /* 
 ** Position a cursor back to the beginning.
 */
@@ -663,6 +757,13 @@ static int cidxFilter(
   const char *zIdxName = 0;
   const char *zAfterKey = 0;
 
+  sqlite3_free(pCsr->zIdxName);
+  pCsr->zIdxName = 0;
+  sqlite3_free(pCsr->zAfterKey);
+  pCsr->zAfterKey = 0;
+  sqlite3_finalize(pCsr->pStmt);
+  pCsr->pStmt = 0;
+
   if( argc>0 ){
     zIdxName = (const char*)sqlite3_value_text(argv[0]);
     if( argc>1 ){
@@ -671,72 +772,13 @@ static int cidxFilter(
   }
 
   if( zIdxName ){
-    char *zTab = 0;
-    char *zCurrentKey = 0;
-    char *zOrderBy = 0;
-    char *zSubWhere = 0;
-    char *zSubExpr = 0;
-    char *zSrcList = 0;
-
-    char **azAfter = 0;
-    CidxIndex *pIdx = 0;
-
-    rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab);
-
-    zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY);
-    zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
-    zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
-    zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
-    zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL);
-
-    if( rc==SQLITE_OK && zAfterKey ){
-      rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
-    }
-
-    if( rc || zAfterKey==0 ){
-      pCsr->pStmt = cidxPrepare(&rc, pCsr, 
-          "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
-          "FROM (SELECT %s FROM %Q ORDER BY %s) AS i",
-          zSubExpr, zTab, zSubWhere, zCurrentKey, 
-          zSrcList, zTab, zOrderBy
-      );
-      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
-    }else{
-      const char *zSep = "";
-      char *zSql;
-      int i;
-
-      zSql = cidxMprintf(&rc, 
-          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
-          zSubExpr, zTab, zSubWhere, zCurrentKey
-      );
-      for(i=pIdx->nCol-1; i>=0; i--){
-        int j;
-        if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
-        for(j=0; j<2; j++){
-          char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
-          zSql = cidxMprintf(&rc, "%z"
-              "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
-              zSql, zSep, zSrcList, zTab, zWhere, zOrderBy
-          );
-          zSep = " UNION ALL ";
-          if( pIdx->aCol[i].bDesc==0 ) break;
-        }
-      }
-      zSql = cidxMprintf(&rc, "%z) AS i", zSql);
-
-      /* printf("SQL: %s\n", zSql); */
+    char *zSql = 0;
+    pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName);
+    pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0;
+    rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql);
+    if( zSql ){
       pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql);
     }
-
-    sqlite3_free(zTab);
-    sqlite3_free(zCurrentKey);
-    sqlite3_free(zOrderBy);
-    sqlite3_free(zSubWhere);
-    sqlite3_free(zSubExpr);
-    sqlite3_free(zSrcList);
-    cidxFreeIndex(pIdx);
-    sqlite3_free(azAfter);
   }
 
   if( pCsr->pStmt ){
@@ -756,26 +798,46 @@ static int cidxColumn(
   int iCol
 ){
   CidxCursor *pCsr = (CidxCursor*)pCursor;
-  assert( iCol>=IIC_ERRMSG && iCol<=IIC_AFTER_KEY );
-  if( iCol==IIC_ERRMSG ){
-    const char *zVal = 0;
-    if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){
-      if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){
-        zVal = "row data mismatch";
+  assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL );
+  switch( iCol ){
+    case IIC_ERRMSG: {
+      const char *zVal = 0;
+      if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){
+        if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){
+          zVal = "row data mismatch";
+        }
+      }else{
+        zVal = "row missing";
       }
-    }else{
-      zVal = "row missing";
+      sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC);
+      break;
+    }
+    case IIC_CURRENT_KEY: {
+      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1));
+      break;
+    }
+    case IIC_INDEX_NAME: {
+      sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT);
+      break;
+    }
+    case IIC_AFTER_KEY: {
+      sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT);
+      break;
+    }
+    case IIC_SCANNER_SQL: {
+      char *zSql = 0;
+      cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql);
+      sqlite3_result_text(ctx, zSql, -1, sqlite3_free);
+      break;
     }
-    sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC);
-  }else if( iCol==IIC_CURRENT_KEY ){
-    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1));
   }
   return SQLITE_OK;
 }
 
 /* Return the ROWID for the sqlite_btreeinfo table */
 static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
-  *pRowid = 0;
+  CidxCursor *pCsr = (CidxCursor*)pCursor;
+  *pRowid = pCsr->iRowid;
   return SQLITE_OK;
 }
 
diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test
index 744a21c69e..d4abdcea7e 100644
--- a/ext/repair/test/checkindex01.test
+++ b/ext/repair/test/checkindex01.test
@@ -43,14 +43,24 @@ proc do_index_check_test {tn idx res} {
 }
 
 
-do_execsql_test 1.2 {
-  SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
+do_execsql_test 1.2.1 {
+  SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1');
 } {
-  1 'five',5
-  1 'four',4
-  1 'one',1
-  1 'three',3
-  1 'two',2
+  1 1 'five',5
+  2 1 'four',4
+  3 1 'one',1
+  4 1 'three',3
+  5 1 'two',2
+}
+do_execsql_test 1.2.2 {
+  SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql
+    FROM incremental_index_check('i1') LIMIT 1;
+} {
+  1
+  'five',5
+  i1
+  {}
+  {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' ORDER BY 1,2) AS i}
 }
 
 do_index_check_test 1.3 i1 {
diff --git a/manifest b/manifest
index f1f80d7a12..d1fed8a386 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\scompiler\swarning\sfrom\sXcode\s9.1.
-D 2017-11-06T09:34:45.112
+C Enhance\sthe\scheckindex.c\svirtual\stable\sso\sthat\sit\swill\soutput\sthe\nindex_name\sand\safter_key\sparameters.\s\sAlso\sadd\sa\snew\sdiagnostic\soutput\ncolumn\snamed\sscanner_sql\swhich\sshows\sthe\sSQL\sstatement\sused\sto\simplement\nthe\scurrent\sindex\sscan.
+D 2017-11-07T16:23:24.445
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -328,12 +328,12 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd
-F ext/repair/checkindex.c 73f26fc1e2e17d68ede5db5b0aaf4869f2d6182f45f3d3624befc503c0f04a70
+F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67
 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f
 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
 F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69
-F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14
+F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 51ee5188b03c4b9508e94afaee4bf1f224aef28875efabda8ce09a5ab641d99e
-R ca8ae1ded89ca445023d13a31194e801
+P 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54
+R 9707d9dec21c954b720b6c37687c0a3c
 U drh
-Z c36ef70e8d55fddaef51df49c2bfea5f
+Z a2de00e6bfce7f49b15935d74b6b30ce
diff --git a/manifest.uuid b/manifest.uuid
index 0cd36e32cb..938db32b6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54
\ No newline at end of file
+32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87
\ No newline at end of file

From 0714a0dee19d2b72f6247fe59cf7c76f316eb4ca Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 7 Nov 2017 16:54:20 +0000
Subject: [PATCH 258/270] Add the --trace option to the sqlite3_checker utility
 program.

FossilOrigin-Name: dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d
---
 ext/repair/sqlite3_checker.tcl | 35 ++++++++++++++++++++++++++--------
 manifest                       | 12 ++++++------
 manifest.uuid                  |  2 +-
 3 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl
index ab8d05e600..88c265f93c 100644
--- a/ext/repair/sqlite3_checker.tcl
+++ b/ext/repair/sqlite3_checker.tcl
@@ -42,7 +42,7 @@ proc tclsh {} {
 
 # Do an incremental integrity check of a single index
 #
-proc check_index {idxname batchsize} {
+proc check_index {idxname batchsize bTrace} {
   set i 0
   set more 1
   set nerr 0
@@ -51,18 +51,30 @@ proc check_index {idxname batchsize} {
                     WHERE name=$idxname}]
   puts -nonewline "$idxname: $i of $max rows ($pct%)\r"
   flush stdout
-  while {$more} {
-    set more 0
-    db eval {SELECT errmsg, current_key AS key
+  if {$bTrace} {
+    set sql {SELECT errmsg, current_key AS key,
+                    CASE WHEN rowid=1 THEN scanner_sql END AS traceOut
                FROM incremental_index_check($idxname)
               WHERE after_key=$key
-              LIMIT $batchsize} {
+              LIMIT $batchsize}
+  } else {
+    set sql {SELECT errmsg, current_key AS key, NULL AS traceOut
+               FROM incremental_index_check($idxname)
+              WHERE after_key=$key
+              LIMIT $batchsize}
+  }
+  while {$more} {
+    set more 0
+    db eval $sql {
       set more 1
       if {$errmsg!=""} {
         incr nerr
         puts "$idxname: key($key): $errmsg"
+      } elseif {$traceOut!=""} {
+        puts "$idxname: $traceOut"
       }
       incr i
+      
     }
     set x [format {%.1f} [expr {($i*100.0)/$max}]]
     if {$x!=$pct} {
@@ -97,6 +109,8 @@ Options:
 
    --tclsh           Run the built-in TCL interpreter (for debugging)
 
+   --trace           (Debugging only:) Output trace information on the scan
+
    --version         Show the version number of SQLite
 }
   exit 1
@@ -110,6 +124,7 @@ set zIndex {}
 set zTable {}
 set batchsize 1000
 set bAll 1
+set bTrace 0
 set argc [llength $argv]
 for {set i 0} {$i<$argc} {incr i} {
   set arg [lindex $argv $i]
@@ -133,6 +148,10 @@ for {set i 0} {$i<$argc} {incr i} {
     set bAll 0
     continue
   }
+  if {[regexp {^-+trace$} $arg]} {
+    set bTrace 1
+    continue
+  }
   if {[regexp {^-+batchsize$} $arg]} {
     incr i
     if {$i>=$argc} {
@@ -224,13 +243,13 @@ if {$bSummary} {
   }
 }
 if {$zIndex!=""} {
-  check_index $zIndex $batchsize
+  check_index $zIndex $batchsize $bTrace
 }
 if {$zTable!=""} {
   foreach idx [db eval {SELECT name FROM sqlite_master
                          WHERE type='index' AND rootpage>0
                            AND tbl_name=$zTable}] {
-    check_index $idx $batchsize
+    check_index $idx $batchsize $bTrace
   }
 }
 if {$bAll} {
@@ -238,6 +257,6 @@ if {$bAll} {
                         WHERE type='index' AND rootpage>0
                         ORDER BY nEntry}]
   foreach idx $allidx {
-    check_index $idx $batchsize
+    check_index $idx $batchsize $bTrace
   }
 }
diff --git a/manifest b/manifest
index d1fed8a386..9879a2fe85 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\scheckindex.c\svirtual\stable\sso\sthat\sit\swill\soutput\sthe\nindex_name\sand\safter_key\sparameters.\s\sAlso\sadd\sa\snew\sdiagnostic\soutput\ncolumn\snamed\sscanner_sql\swhich\sshows\sthe\sSQL\sstatement\sused\sto\simplement\nthe\scurrent\sindex\sscan.
-D 2017-11-07T16:23:24.445
+C Add\sthe\s--trace\soption\sto\sthe\ssqlite3_checker\sutility\sprogram.
+D 2017-11-07T16:54:20.340
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469
 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd
 F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67
 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f
-F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337
+F ext/repair/sqlite3_checker.tcl cc69e7fbc163f94da4a6400609be001543442d9f8f57a797d1eeb7b897585730
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
 F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69
 F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 66d98310b91c69fd01c6a9a958ef1eabda14ec6cd0e4b6612f877f2dfe486c54
-R 9707d9dec21c954b720b6c37687c0a3c
+P 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87
+R d811e2909e0aaa74e9aeec168920f2dd
 U drh
-Z a2de00e6bfce7f49b15935d74b6b30ce
+Z c6a691c1b309b32f8cd47b4317d437a7
diff --git a/manifest.uuid b/manifest.uuid
index 938db32b6b..c71454aad5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87
\ No newline at end of file
+dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d
\ No newline at end of file

From 85e1f46eb93b58143dd8c3849b4d729694258fde Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 18:20:15 +0000
Subject: [PATCH 259/270] Fix a problem causing an INDEXED BY specifying an
 unusable partial index to be mishandled.

FossilOrigin-Name: 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468
---
 manifest            | 16 ++++++++--------
 manifest.uuid       |  2 +-
 src/where.c         | 12 +++++-------
 test/indexedby.test | 21 +++++++++++++++++++++
 4 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 9879a2fe85..e2118d2f81 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s--trace\soption\sto\sthe\ssqlite3_checker\sutility\sprogram.
-D 2017-11-07T16:54:20.340
+C Fix\sa\sproblem\scausing\san\sINDEXED\sBY\sspecifying\san\sunusable\spartial\sindex\sto\sbe\nmishandled.
+D 2017-11-07T18:20:15.108
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
-F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
+F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -959,7 +959,7 @@ F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7
 F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
-F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985
+F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609
 F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
 F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 32e2520ce91351acceda845d81c9567f7a634257dc2b5b90fe6fb6583d8c0f87
-R d811e2909e0aaa74e9aeec168920f2dd
-U drh
-Z c6a691c1b309b32f8cd47b4317d437a7
+P dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d
+R da22ae68c1c6ec503942cfb34af1b16b
+U dan
+Z 75bfd7248538a99270ce42b4a5b4456f
diff --git a/manifest.uuid b/manifest.uuid
index c71454aad5..f26af75224 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d
\ No newline at end of file
+292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 4c1c332d2a..5545a45e87 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2869,9 +2869,11 @@ static int whereLoopAddBtree(
   }
 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
 
-  /* Loop over all indices
-  */
-  for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){
+  /* Loop over all indices. If there was an INDEXED BY clause, then only 
+  ** consider index pProbe.  */
+  for(; rc==SQLITE_OK && pProbe; 
+      pProbe=(pSrc->pIBIndex ? 0 : pProbe->pNext), iSortIdx++
+  ){
     if( pProbe->pPartIdxWhere!=0
      && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){
       testcase( pNew->iTab!=pSrc->iCursor );  /* See ticket [98d973b8f5] */
@@ -2981,10 +2983,6 @@ static int whereLoopAddBtree(
     pBuilder->nRecValid = 0;
     pBuilder->pRec = 0;
 #endif
-
-    /* If there was an INDEXED BY clause, then only that one index is
-    ** considered. */
-    if( pSrc->pIBIndex ) break;
   }
   return rc;
 }
diff --git a/test/indexedby.test b/test/indexedby.test
index 83c7a5cccc..a0f7bea76a 100644
--- a/test/indexedby.test
+++ b/test/indexedby.test
@@ -364,4 +364,25 @@ do_eqp_test 11.10 {
   SELECT a,b,c FROM x2 INDEXED BY x2i WHERE a=1 AND b=1 AND c='3.0';
 } {0 0 0 {SEARCH TABLE x2 USING COVERING INDEX x2i (a=? AND b=? AND rowid=?)}}
 
+#-------------------------------------------------------------------------
+# Check INDEXED BY works (throws an exception) with partial indexes that 
+# cannot be used.
+do_execsql_test 12.1 {
+  CREATE TABLE o1(x INTEGER PRIMARY KEY, y, z);
+  CREATE INDEX p1 ON o1(z);
+  CREATE INDEX p2 ON o1(y) WHERE z=1;
+}
+do_catchsql_test 12.2 {
+  SELECT * FROM o1 INDEXED BY p2 ORDER BY 1;
+} {1 {no query solution}}
+do_execsql_test 12.3 {
+  DROP INDEX p1;
+  DROP INDEX p2;
+  CREATE INDEX p2 ON o1(y) WHERE z=1;
+  CREATE INDEX p1 ON o1(z);
+}
+do_catchsql_test 12.4 {
+  SELECT * FROM o1 INDEXED BY p2 ORDER BY 1;
+} {1 {no query solution}}
+
 finish_test

From 0e90ad6a374f2747d54c8addad5a4a66b5401ed8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 19:02:00 +0000
Subject: [PATCH 260/270] Fix handling of partial indexes in checkindex.c
 (sqlite3_checker).

FossilOrigin-Name: 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231
---
 ext/repair/checkindex.c           | 117 ++++++++++++++++++++----------
 ext/repair/test/checkindex01.test |  32 +++++++-
 manifest                          |  14 ++--
 manifest.uuid                     |   2 +-
 4 files changed, 116 insertions(+), 49 deletions(-)

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index 46ecc12fc4..58706c1aab 100644
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -59,6 +59,7 @@ struct CidxColumn {
 
 typedef struct CidxIndex CidxIndex;
 struct CidxIndex {
+  char *zWhere;                   /* WHERE clause, if any */
   int nCol;                       /* Elements in aCol[] array */
   CidxColumn aCol[1];             /* Array of indexed columns */
 };
@@ -293,6 +294,7 @@ static void cidxFreeIndex(CidxIndex *pIdx){
     for(i=0; i<pIdx->nCol; i++){
       sqlite3_free(pIdx->aCol[i].zExpr);
     }
+    sqlite3_free(pIdx->zWhere);
     sqlite3_free(pIdx);
   }
 }
@@ -312,6 +314,24 @@ static int cidx_isident(char c){
 #define CIDX_PARSE_OPEN  2      /*  "(" */
 #define CIDX_PARSE_CLOSE 3      /*  ")" */
 
+/*
+** Argument zIn points into the start, middle or end of a CREATE INDEX
+** statement. If argument pbDoNotTrim is non-NULL, then this function
+** scans the input until it finds EOF, a comma (",") or an open or
+** close parenthesis character. It then sets (*pzOut) to point to said
+** character and returns a CIDX_PARSE_XXX constant as appropriate. The
+** parser is smart enough that special characters inside SQL strings
+** or comments are not returned for.
+**
+** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut
+** to point to the first character of the string that is not whitespace
+** or part of an SQL comment and returns CIDX_PARSE_EOF.
+**
+** Additionally, if pbDoNotTrim is not NULL and the element immediately
+** before (*pzOut) is an SQL comment of the form "-- comment", then
+** (*pbDoNotTrim) is set before returning. In all other cases it is
+** cleared.
+*/
 static int cidxFindNext(
   const char *zIn, 
   const char **pzOut,
@@ -320,6 +340,7 @@ static int cidxFindNext(
   const char *z = zIn;
 
   while( 1 ){
+    while( cidx_isspace(*z) ) z++;
     if( z[0]=='-' && z[1]=='-' ){
       z += 2;
       while( z[0]!='\n' ){
@@ -327,9 +348,18 @@ static int cidxFindNext(
         z++;
       }
       while( cidx_isspace(*z) ) z++;
-      *pbDoNotTrim = 1;
+      if( pbDoNotTrim ) *pbDoNotTrim = 1;
+    }else
+    if( z[0]=='/' && z[1]=='*' ){
+      z += 2;
+      while( z[0]!='*' || z[1]!='/' ){
+        if( z[1]=='\0' ) return CIDX_PARSE_EOF;
+        z++;
+      }
+      z += 2;
     }else{
       *pzOut = z;
+      if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF;
       switch( *z ){
         case '\0':
           return CIDX_PARSE_EOF;
@@ -359,17 +389,6 @@ static int cidxFindNext(
           while( *z++!=']' );
           break;
   
-        case '/':
-          if( z[1]=='*' ){
-            z += 2;
-            while( z[0]!='*' || z[1]!='/' ){
-              if( z[1]=='\0' ) return CIDX_PARSE_EOF;
-              z++;
-            }
-            z += 2;
-            break;
-          }
-  
         default:
           z++;
           break;
@@ -424,6 +443,14 @@ static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){
     z++;
   }
 
+  /* Search for a WHERE clause */
+  cidxFindNext(z, &z, 0);
+  if( 0==sqlite3_strnicmp(z, "where", 5) ){
+    pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]);
+  }else if( z[0]!='\0' ){
+    goto parse_error;
+  }
+
   return rc;
 
  parse_error:
@@ -477,6 +504,7 @@ static int cidxLookupIndex(
           p->zExpr = 0;
         }
         pIdx->nCol = iCol;
+        pIdx->zWhere = 0;
       }
       cidxFinalize(&rc, pInfo);
     }
@@ -700,36 +728,45 @@ int cidxGenerateScanSql(
     rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
   }
 
-  if( rc || zAfterKey==0 ){
-    *pzSqlOut = cidxMprintf(&rc,
-        "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
-        "FROM (SELECT %s FROM %Q ORDER BY %s) AS i",
-        zSubExpr, zTab, zSubWhere, zCurrentKey, 
-        zSrcList, zTab, zOrderBy
-    );
-  }else{
-    const char *zSep = "";
-    char *zSql;
-    int i;
-
-    zSql = cidxMprintf(&rc, 
-        "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
-        zSubExpr, zTab, zSubWhere, zCurrentKey
-    );
-    for(i=pIdx->nCol-1; i>=0; i--){
-      int j;
-      if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
-      for(j=0; j<2; j++){
-        char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
-        zSql = cidxMprintf(&rc, "%z"
-            "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
-            zSql, zSep, zSrcList, zTab, zWhere, zOrderBy
-        );
-        zSep = " UNION ALL ";
-        if( pIdx->aCol[i].bDesc==0 ) break;
+  if( rc==SQLITE_OK ){
+    if( zAfterKey==0 ){
+      *pzSqlOut = cidxMprintf(&rc,
+          "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
+          "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i",
+          zSubExpr, zTab, zSubWhere, zCurrentKey, 
+          zSrcList, zTab, zIdxName, 
+          (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""),
+          zOrderBy
+      );
+    }else{
+      const char *zSep = "";
+      char *zSql;
+      int i;
+  
+      zSql = cidxMprintf(&rc, 
+          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+          zSubExpr, zTab, zSubWhere, zCurrentKey
+      );
+      for(i=pIdx->nCol-1; i>=0; i--){
+        int j;
+        if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
+        for(j=0; j<2; j++){
+          char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
+          zSql = cidxMprintf(&rc, "%z"
+              "%sSELECT * FROM ("
+                "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s"
+              ")",
+              zSql, zSep, zSrcList, zTab, zIdxName, 
+              pIdx->zWhere ? pIdx->zWhere : "",
+              pIdx->zWhere ? " AND " : "",
+              zWhere, zOrderBy
+          );
+          zSep = " UNION ALL ";
+          if( pIdx->aCol[i].bDesc==0 ) break;
+        }
       }
+      *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql);
     }
-    *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql);
   }
 
   sqlite3_free(zTab);
diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test
index d4abdcea7e..25930397d3 100644
--- a/ext/repair/test/checkindex01.test
+++ b/ext/repair/test/checkindex01.test
@@ -60,7 +60,7 @@ do_execsql_test 1.2.2 {
   'five',5
   i1
   {}
-  {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' ORDER BY 1,2) AS i}
+  {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i}
 }
 
 do_index_check_test 1.3 i1 {
@@ -319,3 +319,33 @@ do_index_check_test 6.2 t6x3 {
   {} 3,2,1 
   {} 6,5,4
 }
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+  CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z);
+  INSERT INTO t7 VALUES(1, 1, 1);
+  INSERT INTO t7 VALUES(2, 2, 0);
+  INSERT INTO t7 VALUES(3, 3, 1);
+  INSERT INTO t7 VALUES(4, 4, 0);
+
+  CREATE INDEX t7i1 ON t7(y) WHERE z=1;
+  CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1;
+  CREATE INDEX t7i3 ON t7(y) WHERE -- yep 
+  z=1;
+  CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep;
+}
+do_index_check_test 7.1 t7i1 {
+  {} 1,1 {} 3,3
+}
+do_index_check_test 7.2 t7i2 {
+  {} 1,1 {} 3,3
+}
+do_index_check_test 7.3 t7i3 {
+  {} 1,1 {} 3,3
+}
+do_index_check_test 7.4 t7i4 {
+  {} 1,1 {} 3,3
+}
+
+
diff --git a/manifest b/manifest
index e2118d2f81..004ac6c071 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\san\sINDEXED\sBY\sspecifying\san\sunusable\spartial\sindex\sto\sbe\nmishandled.
-D 2017-11-07T18:20:15.108
+C Fix\shandling\sof\spartial\sindexes\sin\scheckindex.c\s(sqlite3_checker).
+D 2017-11-07T19:02:00.173
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -328,12 +328,12 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd
-F ext/repair/checkindex.c 50264729469ae44abe9ce6c78944ec577e50faf26e362f3036142425807a8b67
+F ext/repair/checkindex.c 7d28c01a2e012ac64257d230fc452b2cafb78311a91a343633d01d95220f66f3
 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f
 F ext/repair/sqlite3_checker.tcl cc69e7fbc163f94da4a6400609be001543442d9f8f57a797d1eeb7b897585730
 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e
 F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69
-F ext/repair/test/checkindex01.test c4c9e62b1068ddb2d8b5c75580c3036713fc27ec105bbc579114da4ab29d53f4
+F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dc217b7cfe680044d8742e317701abd0269162da8f5cb097361ae7f47fd9ba2d
-R da22ae68c1c6ec503942cfb34af1b16b
+P 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468
+R 34ab99e7735ca564d873e9e5cddc35ee
 U dan
-Z 75bfd7248538a99270ce42b4a5b4456f
+Z e9b579d9d018f181ffb82f5d81fd8f28
diff --git a/manifest.uuid b/manifest.uuid
index f26af75224..ab0b7bdaaf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468
\ No newline at end of file
+31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231
\ No newline at end of file

From 592eca1fda4a090df0b4e7107799fa07a45a519e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Nov 2017 02:50:09 +0000
Subject: [PATCH 261/270] Improved comments used for documentation of
 sqlite3_vfs.  No changes to code.

FossilOrigin-Name: db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 18 ++++++++++++------
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 004ac6c071..16b3db4323 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\shandling\sof\spartial\sindexes\sin\scheckindex.c\s(sqlite3_checker).
-D 2017-11-07T19:02:00.173
+C Improved\scomments\sused\sfor\sdocumentation\sof\ssqlite3_vfs.\s\sNo\schanges\nto\scode.
+D 2017-11-08T02:50:09.495
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -472,7 +472,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
-F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
+F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 292a04086a902634fc514b379a2b245eb2681c1b84d9bb950b6ecb9aab28b468
-R 34ab99e7735ca564d873e9e5cddc35ee
-U dan
-Z e9b579d9d018f181ffb82f5d81fd8f28
+P 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231
+R 3a2f607d35fb53b6e561d687398b02f3
+U drh
+Z 38299a5cdd3519065be7d21e475b6bdf
diff --git a/manifest.uuid b/manifest.uuid
index ab0b7bdaaf..b7832db39a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231
\ No newline at end of file
+db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 9ff366304f..f2a379d55a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -1131,12 +1131,18 @@ typedef struct sqlite3_api_routines sqlite3_api_routines;
 ** in the name of the object stands for "virtual file system".  See
 ** the [VFS | VFS documentation] for further information.
 **
-** The value of the iVersion field is initially 1 but may be larger in
-** future versions of SQLite.  Additional fields may be appended to this
-** object when the iVersion value is increased.  Note that the structure
-** of the sqlite3_vfs object changes in the transaction between
-** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not
-** modified.
+** The VFS interface is sometimes extended by adding new methods onto
+** the end.  Each time such an extension occurs, the iVersion field
+** is incremented.  The iVersion value started out as 1 in
+** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2
+** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased
+** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6].  Additional fields
+** may be appended to the sqlite3_vfs object and the iVersion value
+** may increase again in future versions of SQLite.
+** Note that the structure
+** of the sqlite3_vfs object changes in the transition from
+** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0]
+** and yet the iVersion field was not modified.
 **
 ** The szOsFile field is the size of the subclassed [sqlite3_file]
 ** structure used by this VFS.  mxPathname is the maximum length of

From 3e846cb81ff05c85c5a8e1a982a11f6b6d5d7764 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 8 Nov 2017 11:14:53 +0000
Subject: [PATCH 262/270] Fix a problem causing LSM to add unnecessary padding
 to empty segments in compressed databases.

FossilOrigin-Name: 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
---
 ext/lsm1/Makefile   |  4 ++--
 ext/lsm1/lsm_file.c |  2 +-
 manifest            | 16 ++++++++--------
 manifest.uuid       |  2 +-
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile
index a4f8ebd367..7022b5682c 100644
--- a/ext/lsm1/Makefile
+++ b/ext/lsm1/Makefile
@@ -43,7 +43,7 @@ LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
 
 # all: lsm.so
 
-LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
+LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
 
 lsm.so:	$(LSMOBJ)
 	$(TCCX) -shared -o lsm.so $(LSMOBJ)
@@ -53,4 +53,4 @@ lsm.so:	$(LSMOBJ)
 	
 lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
 	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
-	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
+	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz
diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c
index 3283c023dd..4001aac978 100644
--- a/ext/lsm1/lsm_file.c
+++ b/ext/lsm1/lsm_file.c
@@ -2803,7 +2803,7 @@ int lsmFsSortedPadding(
   Segment *pSeg
 ){
   int rc = LSM_OK;
-  if( pFS->pCompress ){
+  if( pFS->pCompress && pSeg->iFirst ){
     Pgno iLast2;
     Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
     int nPad;                       /* Bytes of padding required */
diff --git a/manifest b/manifest
index 16b3db4323..22a536158c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\scomments\sused\sfor\sdocumentation\sof\ssqlite3_vfs.\s\sNo\schanges\nto\scode.
-D 2017-11-08T02:50:09.495
+C Fix\sa\sproblem\scausing\sLSM\sto\sadd\sunnecessary\spadding\sto\sempty\ssegments\sin\ncompressed\sdatabases.
+D 2017-11-08T11:14:53.660
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -211,7 +211,7 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c 635775226d07c743c770888a9dd5175afc6e67d3e28a4032b7fedc3bcaa92e65
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
-F ext/lsm1/Makefile 2951812df1c1cbc9e023af7e070876f479b3d75ce3898b3b9d00f83fecf13608
+F ext/lsm1/Makefile 98b0a24b45e248283d6bea4b6cb3e58d7b394edd8e96a0ac28c5fa5104813bad
 F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013
 F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86
 F ext/lsm1/lsm-test/lsmtest.h 5847594d4b43ec3412e1fd97104f7eb5fd770be55e691e6cb2e80929f86bebe3
@@ -240,7 +240,7 @@ F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f52
 F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001
 F ext/lsm1/lsmInt.h e9e5c5f08e35a104086102b3def94ee69cbc0d39002f6596f5c80a640439628e
 F ext/lsm1/lsm_ckpt.c ac6fb4581983291c2e0be6fbb68f12b26f0c08d606835c05417be1323d0fdd03
-F ext/lsm1/lsm_file.c e50f0e15427513dca4507110d0107268544bcaebb0ff0599a822da1f8d0da9eb
+F ext/lsm1/lsm_file.c 4b3fb56336fbc9d941e1b2042e809d986feebdc41e73dc7fc4fdc0dd1bd4274d
 F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c
 F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 31932a9eb8dbb33d5535715ae8bbfdc55ce66b1a1a0abd57cefe720eeb31e231
-R 3a2f607d35fb53b6e561d687398b02f3
-U drh
-Z 38299a5cdd3519065be7d21e475b6bdf
+P db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8
+R 6c8e575584f16f8416ee1b9774368bff
+U dan
+Z 515295e5c95123e6255cf7559983f082
diff --git a/manifest.uuid b/manifest.uuid
index b7832db39a..bac99cd8b8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8
\ No newline at end of file
+1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
\ No newline at end of file

From 2fba394c97f9d9cf6b6c447dd5c2d01651593c73 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 9 Nov 2017 03:55:09 +0000
Subject: [PATCH 263/270] Disallow ORDER BY and LIMIT on UPDATE and DELETE of
 views and WITHOUT ROWID tables.  This is a temporary fix for ticket
 [d4beea1633f1b88f] until a better solution can be found.

FossilOrigin-Name: 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
---
 manifest             | 16 ++++++++--------
 manifest.uuid        |  2 +-
 src/resolve.c        |  6 +++++-
 test/wherelimit.test | 37 +++++++++++++++++++++++++++++++++++++
 4 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 22a536158c..9a2dbc0bc9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\sLSM\sto\sadd\sunnecessary\spadding\sto\sempty\ssegments\sin\ncompressed\sdatabases.
-D 2017-11-08T11:14:53.660
+C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound.
+D 2017-11-09T03:55:09.531
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -468,7 +468,7 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
+F src/resolve.c 704978e5aabb9f524789e3b53016b4068a4e64c669c0f8ff025d2b23b95d62b5
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
@@ -1557,7 +1557,7 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
-F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
+F test/wherelimit.test baa8e5e30b2eddc395cbb3edb313deea14ca6938ea8a96c02a03c34d0ab84b5b
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P db8f22d069b1725f16b208c96c0c47b6ab0e30236feeac43cca4217ef80316b8
-R 6c8e575584f16f8416ee1b9774368bff
-U dan
-Z 515295e5c95123e6255cf7559983f082
+P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
+R a028f3be0864f739aeb3787daf46fc8e
+U drh
+Z 97937a65b1ac3d4b961579eac7d25cbd
diff --git a/manifest.uuid b/manifest.uuid
index bac99cd8b8..e270b8e48e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
\ No newline at end of file
+62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
\ No newline at end of file
diff --git a/src/resolve.c b/src/resolve.c
index 4a1e8284c8..945654ead0 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -596,7 +596,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       SrcList *pSrcList = pNC->pSrcList;
       struct SrcList_item *pItem;
       assert( pSrcList && pSrcList->nSrc==1 );
-      pItem = pSrcList->a; 
+      pItem = pSrcList->a;
+      if( !HasRowid(pItem->pTab) || pItem->pTab->pSelect!=0 ){
+         sqlite3ErrorMsg(pParse, "ORDER BY and LIMIT not support for table %s",
+                                 pItem->pTab->zName);
+      }
       pExpr->op = TK_COLUMN;
       pExpr->pTab = pItem->pTab;
       pExpr->iTable = pItem->iCursor;
diff --git a/test/wherelimit.test b/test/wherelimit.test
index f0cfbb61e8..e10d90b40b 100644
--- a/test/wherelimit.test
+++ b/test/wherelimit.test
@@ -279,6 +279,43 @@ ifcapable {update_delete_limit} {
     execsql {SELECT count(*) FROM t1 WHERE y=1}
   } {6}
 
+  # Cannot use a LIMIT for UPDATE or DELETE against a WITHOUT ROWID table
+  # or a VIEW.  (We should fix this someday).
+  #
+  db close
+  sqlite3 db :memory:
+  do_execsql_test wherelimit-4.1 {
+    CREATE TABLE t1(a int);
+    INSERT INTO t1 VALUES(1);
+    INSERT INTO t1 VALUES(2);
+    INSERT INTO t1 VALUES(3);
+    CREATE TABLE t2(a int);
+    INSERT INTO t2 SELECT a+100 FROM t1;
+    CREATE VIEW tv(r,a) AS
+       SELECT rowid, a FROM t2 UNION ALL SELECT rowid, a FROM t1;
+    CREATE TRIGGER tv_del INSTEAD OF DELETE ON tv
+    BEGIN
+      DELETE FROM t1 WHERE rowid=old.r;
+      DELETE FROM t2 WHERE rowid=old.r;
+    END;
+  } {}
+  do_catchsql_test wherelimit-4.2 {
+    DELETE FROM tv WHERE 1 LIMIT 2;
+  } {1 {ORDER BY and LIMIT not support for table tv}}
+  do_catchsql_test wherelimit-4.3 {
+    DELETE FROM tv WHERE 1 ORDER BY a LIMIT 2;
+  } {1 {ORDER BY and LIMIT not support for table tv}}
+  do_execsql_test wherelimit-4.10 {
+    CREATE TABLE t3(a,b,c,d TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID;
+    INSERT INTO t3(a,b,c,d) VALUES(1,2,3,4),(5,6,7,8),(9,10,11,12);
+  } {}
+  do_catchsql_test wherelimit-4.11 {
+    DELETE FROM t3 WHERE a=5 LIMIT 2;
+  } {1 {ORDER BY and LIMIT not support for table t3}}
+  do_execsql_test wherelimit-4.12 {
+    SELECT a,b,c,d FROM t3 ORDER BY 1;
+  } {1 2 3 4 5 6 7 8 9 10 11 12}
+
 }
 
 finish_test

From b3c16b899b323c16ef84994359143746a0af2320 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 9 Nov 2017 19:53:06 +0000
Subject: [PATCH 264/270] Add SQLITE_ENABLE_UPDATE_DELETE_LIMIT for views and
 WITHOUT ROWID tables.

FossilOrigin-Name: 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce
---
 manifest              |  26 +++++-----
 manifest.uuid         |   2 +-
 src/delete.c          |  95 +++++++++++++++++++++++++-----------
 src/parse.y           |   7 +--
 src/sqliteInt.h       |   9 ++--
 src/update.c          |  26 ++++++++--
 test/wherelimit.test  |  12 +++--
 test/wherelimit2.test | 109 ++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 232 insertions(+), 54 deletions(-)
 create mode 100644 test/wherelimit2.test

diff --git a/manifest b/manifest
index 9a2dbc0bc9..4f63569478 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound.
-D 2017-11-09T03:55:09.531
+C Add\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfor\sviews\sand\sWITHOUT\sROWID\stables.
+D 2017-11-09T19:53:06.815
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
+F src/delete.c 001653cc4065ad83d47f4307c29572437f778f1fe249d482894e100275d27efd
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
@@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
+F src/parse.y 5fbc16a4adf5da8ee22a1ef271e001fd6b766eebc3195b52ba2c30cecbbaf2d0
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -475,7 +475,7 @@ F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8
 F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
+F src/sqliteInt.h 821479be791acfdb63177b21a491e604526191c1630d1d65d7a4a7c65d203c83
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -535,7 +535,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef
 F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
-F src/update.c 5404be9e840717323a69209190cdbc9d0d34adaedaaf1d1a1069babf2c4171c0
+F src/update.c 5123fcb4aa98a705ced2acb172c1761a570d142d77901b4f4ce38acb191ef8ed
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
@@ -1557,7 +1557,8 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
-F test/wherelimit.test baa8e5e30b2eddc395cbb3edb313deea14ca6938ea8a96c02a03c34d0ab84b5b
+F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8
+F test/wherelimit2.test 501e470c9cf62e96359e3386ae520874792b83322060168ee9c708b2e953b985
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
@@ -1673,7 +1674,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
-R a028f3be0864f739aeb3787daf46fc8e
-U drh
-Z 97937a65b1ac3d4b961579eac7d25cbd
+P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
+R 46792a4c59e5e3fb96fc57893e581fbe
+T *branch * update-delete-limit-fix
+T *sym-update-delete-limit-fix *
+T -sym-trunk *
+U dan
+Z c86d100b98964de2c810aa6a7710c91c
diff --git a/manifest.uuid b/manifest.uuid
index e270b8e48e..e7ae4dab5c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
\ No newline at end of file
+584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index cd78f68f41..509fd8af8d 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -90,6 +90,9 @@ void sqlite3MaterializeView(
   Parse *pParse,       /* Parsing context */
   Table *pView,        /* View definition */
   Expr *pWhere,        /* Optional WHERE clause to be added */
+  ExprList *pOrderBy,  /* Optional ORDER BY clause */
+  Expr *pLimit,        /* Optional LIMIT clause */
+  Expr *pOffset,       /* Optional OFFSET clause */
   int iCur             /* Cursor number for ephemeral table */
 ){
   SelectDest dest;
@@ -106,8 +109,8 @@ void sqlite3MaterializeView(
     assert( pFrom->a[0].pOn==0 );
     assert( pFrom->a[0].pUsing==0 );
   }
-  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 
-                          SF_IncludeHidden, 0, 0);
+  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, 
+                          SF_IncludeHidden, pLimit, pOffset);
   sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
   sqlite3Select(pParse, pSel, &dest);
   sqlite3SelectDelete(db, pSel);
@@ -132,18 +135,23 @@ Expr *sqlite3LimitWhere(
   Expr *pOffset,               /* The OFFSET clause.  May be null */
   char *zStmtType              /* Either DELETE or UPDATE.  For err msgs. */
 ){
-  Expr *pWhereRowid = NULL;    /* WHERE rowid .. */
+  sqlite3 *db = pParse->db;
+  Expr *pLhs = NULL;           /* LHS of IN(SELECT...) operator */
   Expr *pInClause = NULL;      /* WHERE rowid IN ( select ) */
-  Expr *pSelectRowid = NULL;   /* SELECT rowid ... */
   ExprList *pEList = NULL;     /* Expression list contaning only pSelectRowid */
   SrcList *pSelectSrc = NULL;  /* SELECT rowid FROM x ... (dup of pSrc) */
   Select *pSelect = NULL;      /* Complete SELECT tree */
+  Table *pTab;
 
   /* Check that there isn't an ORDER BY without a LIMIT clause.
   */
-  if( pOrderBy && (pLimit == 0) ) {
+  if( pOrderBy && pLimit==0 ) {
     sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType);
-    goto limit_where_cleanup;
+    sqlite3ExprDelete(pParse->db, pWhere);
+    sqlite3ExprListDelete(pParse->db, pOrderBy);
+    sqlite3ExprDelete(pParse->db, pLimit);
+    sqlite3ExprDelete(pParse->db, pOffset);
+    return 0;
   }
 
   /* We only need to generate a select expression if there
@@ -164,36 +172,44 @@ Expr *sqlite3LimitWhere(
   **   );
   */
 
-  pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0);
-  if( pSelectRowid == 0 ) goto limit_where_cleanup;
-  pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid);
-  if( pEList == 0 ) goto limit_where_cleanup;
+  pTab = pSrc->a[0].pTab;
+  if( HasRowid(pTab) ){
+    pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0);
+    pEList = sqlite3ExprListAppend(
+        pParse, 0, sqlite3PExpr(pParse, TK_ROW, 0, 0)
+    );
+  }else{
+    Index *pPk = sqlite3PrimaryKeyIndex(pTab);
+    if( pPk->nKeyCol==1 ){
+      pLhs = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[0]].zName);
+    }else{
+      int i;
+      for(i=0; i<pPk->nKeyCol; i++){
+        Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zName);
+        pEList = sqlite3ExprListAppend(pParse, pEList, p);
+      }
+      pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
+      if( pLhs ){
+        pLhs->x.pList = sqlite3ExprListDup(db, pEList, 0);
+      }
+    }
+  }
 
   /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree
   ** and the SELECT subtree. */
+  pSrc->a[0].pTab = 0;
   pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0);
-  if( pSelectSrc == 0 ) {
-    sqlite3ExprListDelete(pParse->db, pEList);
-    goto limit_where_cleanup;
-  }
+  pSrc->a[0].pTab = pTab;
 
   /* generate the SELECT expression tree. */
-  pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0,
-                             pOrderBy,0,pLimit,pOffset);
-  if( pSelect == 0 ) return 0;
+  pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, 
+      pOrderBy,0,pLimit,pOffset
+  );
 
   /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */
-  pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0);
-  pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0) : 0;
+  pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0);
   sqlite3PExprAddSelect(pParse, pInClause, pSelect);
   return pInClause;
-
-limit_where_cleanup:
-  sqlite3ExprDelete(pParse->db, pWhere);
-  sqlite3ExprListDelete(pParse->db, pOrderBy);
-  sqlite3ExprDelete(pParse->db, pLimit);
-  sqlite3ExprDelete(pParse->db, pOffset);
-  return 0;
 }
 #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */
        /*      && !defined(SQLITE_OMIT_SUBQUERY) */
@@ -205,10 +221,13 @@ limit_where_cleanup:
 **                 \________/       \________________/
 **                  pTabList              pWhere
 */
-void sqlite3DeleteFrom(
+void sqlite3DeleteFromLimit(
   Parse *pParse,         /* The parser context */
   SrcList *pTabList,     /* The table from which we should delete things */
-  Expr *pWhere           /* The WHERE clause.  May be null */
+  Expr *pWhere,          /* The WHERE clause.  May be null */
+  ExprList *pOrderBy,
+  Expr *pLimit,
+  Expr *pOffset
 ){
   Vdbe *v;               /* The virtual database engine */
   Table *pTab;           /* The table from which records will be deleted */
@@ -253,6 +272,7 @@ void sqlite3DeleteFrom(
   }
   assert( pTabList->nSrc==1 );
 
+
   /* Locate the table which we want to delete.  This table has to be
   ** put in an SrcList structure because some of the subroutines we
   ** will be calling are designed to work with multiple tables and expect
@@ -277,6 +297,16 @@ void sqlite3DeleteFrom(
 # define isView 0
 #endif
 
+#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
+  if( !isView ){
+    pWhere = sqlite3LimitWhere(
+        pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "DELETE"
+    );
+    pOrderBy = 0;
+    pLimit = pOffset = 0;
+  }
+#endif
+
   /* If pTab is really a view, make sure it has been initialized.
   */
   if( sqlite3ViewGetColumnNames(pParse, pTab) ){
@@ -324,8 +354,12 @@ void sqlite3DeleteFrom(
   */
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   if( isView ){
-    sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur);
+    sqlite3MaterializeView(pParse, pTab, 
+        pWhere, pOrderBy, pLimit, pOffset, iTabCur
+    );
     iDataCur = iIdxCur = iTabCur;
+    pOrderBy = 0;
+    pLimit = pOffset = 0;
   }
 #endif
 
@@ -569,6 +603,9 @@ delete_from_cleanup:
   sqlite3AuthContextPop(&sContext);
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprDelete(db, pWhere);
+  sqlite3ExprListDelete(db, pOrderBy);
+  sqlite3ExprDelete(db, pLimit);
+  sqlite3ExprDelete(db, pOffset);
   sqlite3DbFree(db, aToOpen);
   return;
 }
diff --git a/src/parse.y b/src/parse.y
index a8d080eed3..928e53dd82 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -753,8 +753,7 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W)
         orderby_opt(O) limit_opt(L). {
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
-  W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE");
-  sqlite3DeleteFrom(pParse,X,W);
+  sqlite3DeleteFromLimit(pParse,X,W,O,L.pLimit,L.pOffset); 
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
@@ -779,8 +778,10 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
+#if 0
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE");
-  sqlite3Update(pParse,X,Y,W,R);
+#endif
+  sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 378b085021..d3c412e222 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3764,8 +3764,11 @@ void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
 Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*);
 #endif
-void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
-void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
+void sqlite3DeleteFromLimit(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*);
+#define sqlite3DeleteFrom(x,y,z) sqlite3DeleteFromLimit(x,y,z,0,0,0)
+void sqlite3UpdateLimit(Parse*, SrcList*, ExprList*, Expr*, int,
+                        ExprList*,Expr*,Expr*);
+#define sqlite3Update(v,w,x,y,z) sqlite3UpdateLimit(v,w,x,y,z,0,0,0)
 WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
 void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
@@ -3889,7 +3892,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3*);
 void sqlite3ChangeCookie(Parse*, int);
 
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
-void sqlite3MaterializeView(Parse*, Table*, Expr*, int);
+void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,Expr*,int);
 #endif
 
 #ifndef SQLITE_OMIT_TRIGGER
diff --git a/src/update.c b/src/update.c
index e69efdb6bf..f63a9ffe8a 100644
--- a/src/update.c
+++ b/src/update.c
@@ -86,12 +86,15 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
 **          \_______/ \________/     \______/       \________________/
 *            onError   pTabList      pChanges             pWhere
 */
-void sqlite3Update(
+void sqlite3UpdateLimit(
   Parse *pParse,         /* The parser context */
   SrcList *pTabList,     /* The table in which we should change things */
   ExprList *pChanges,    /* Things to be changed */
   Expr *pWhere,          /* The WHERE clause.  May be null */
-  int onError            /* How to handle constraint errors */
+  int onError,           /* How to handle constraint errors */
+  ExprList *pOrderBy,
+  Expr *pLimit,
+  Expr *pOffset
 ){
   int i, j;              /* Loop counters */
   Table *pTab;           /* The table to be updated */
@@ -176,6 +179,16 @@ void sqlite3Update(
 # define isView 0
 #endif
 
+#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
+  if( !isView ){
+    pWhere = sqlite3LimitWhere(
+        pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "UPDATE"
+    );
+    pOrderBy = 0;
+    pLimit = pOffset = 0;
+  }
+#endif
+
   if( sqlite3ViewGetColumnNames(pParse, pTab) ){
     goto update_cleanup;
   }
@@ -344,7 +357,11 @@ void sqlite3Update(
   */
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   if( isView ){
-    sqlite3MaterializeView(pParse, pTab, pWhere, iDataCur);
+    sqlite3MaterializeView(pParse, pTab, 
+        pWhere, pOrderBy, pLimit, pOffset, iDataCur
+    );
+    pOrderBy = 0;
+    pLimit = pOffset = 0;
   }
 #endif
 
@@ -728,6 +745,9 @@ update_cleanup:
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprListDelete(db, pChanges);
   sqlite3ExprDelete(db, pWhere);
+  sqlite3ExprListDelete(db, pOrderBy);
+  sqlite3ExprDelete(db, pLimit);
+  sqlite3ExprDelete(db, pOffset);
   return;
 }
 /* Make sure "isView" and other macros defined above are undefined. Otherwise
diff --git a/test/wherelimit.test b/test/wherelimit.test
index e10d90b40b..e2ba18d2c8 100644
--- a/test/wherelimit.test
+++ b/test/wherelimit.test
@@ -38,6 +38,8 @@ proc create_test_data {size} {
 
 ifcapable {update_delete_limit} {
 
+  execsql { CREATE TABLE t1(x, y) }
+
   # check syntax error support
   do_test wherelimit-0.1 {
     catchsql {DELETE FROM t1 ORDER BY x}
@@ -49,6 +51,8 @@ ifcapable {update_delete_limit} {
     catchsql {UPDATE t1 SET y=1 WHERE x=1 ORDER BY x}
   } {1 {ORDER BY without LIMIT on UPDATE}}
 
+  execsql { DROP TABLE t1 }
+
   # no AS on table sources
   do_test wherelimit-0.4 {
     catchsql {DELETE FROM t1 AS a WHERE x=1}
@@ -301,20 +305,20 @@ ifcapable {update_delete_limit} {
   } {}
   do_catchsql_test wherelimit-4.2 {
     DELETE FROM tv WHERE 1 LIMIT 2;
-  } {1 {ORDER BY and LIMIT not support for table tv}}
+  } {0 {}}
   do_catchsql_test wherelimit-4.3 {
     DELETE FROM tv WHERE 1 ORDER BY a LIMIT 2;
-  } {1 {ORDER BY and LIMIT not support for table tv}}
+  } {0 {}}
   do_execsql_test wherelimit-4.10 {
     CREATE TABLE t3(a,b,c,d TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID;
     INSERT INTO t3(a,b,c,d) VALUES(1,2,3,4),(5,6,7,8),(9,10,11,12);
   } {}
   do_catchsql_test wherelimit-4.11 {
     DELETE FROM t3 WHERE a=5 LIMIT 2;
-  } {1 {ORDER BY and LIMIT not support for table t3}}
+  } {0 {}}
   do_execsql_test wherelimit-4.12 {
     SELECT a,b,c,d FROM t3 ORDER BY 1;
-  } {1 2 3 4 5 6 7 8 9 10 11 12}
+  } {1 2 3 4 9 10 11 12}
 
 }
 
diff --git a/test/wherelimit2.test b/test/wherelimit2.test
new file mode 100644
index 0000000000..e505da42bb
--- /dev/null
+++ b/test/wherelimit2.test
@@ -0,0 +1,109 @@
+# 2008 October 6
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the LIMIT ... OFFSET ... clause
+#  of UPDATE and DELETE statements.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix wherelimit2
+
+ifcapable !update_delete_limit {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'f');
+  INSERT INTO t1 VALUES(2, 'e');
+  INSERT INTO t1 VALUES(3, 'd');
+  INSERT INTO t1 VALUES(4, 'c');
+  INSERT INTO t1 VALUES(5, 'b');
+  INSERT INTO t1 VALUES(6, 'a');
+
+  CREATE VIEW v1 AS SELECT a,b FROM t1;
+  CREATE TABLE log(op, a);
+
+  CREATE TRIGGER v1del INSTEAD OF DELETE ON v1 BEGIN
+    INSERT INTO log VALUES('delete', old.a);
+  END;
+
+  CREATE TRIGGER v1upd INSTEAD OF UPDATE ON v1 BEGIN
+    INSERT INTO log VALUES('update', old.a);
+  END;
+}
+
+do_execsql_test 1.1 {
+  DELETE FROM v1 ORDER BY a LIMIT 3;
+  SELECT * FROM log; DELETE FROM log;
+} {
+  delete 1 delete 2 delete 3
+}
+do_execsql_test 1.2 {
+  DELETE FROM v1 ORDER BY b LIMIT 3;
+  SELECT * FROM log; DELETE FROM log;
+} {
+  delete 6 delete 5 delete 4
+}
+do_execsql_test 1.3 {
+  UPDATE v1 SET b = 555 ORDER BY a LIMIT 3;
+  SELECT * FROM log; DELETE FROM log;
+} {
+  update 1 update 2 update 3
+}
+do_execsql_test 1.4 {
+  UPDATE v1 SET b = 555 ORDER BY b LIMIT 3;
+  SELECT * FROM log; DELETE FROM log;
+} {
+  update 6 update 5 update 4
+}
+
+do_execsql_test 2.0 {
+  CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
+  INSERT INTO t2 VALUES(1, 1, 'h');
+  INSERT INTO t2 VALUES(1, 2, 'g');
+  INSERT INTO t2 VALUES(2, 1, 'f');
+  INSERT INTO t2 VALUES(2, 2, 'e');
+  INSERT INTO t2 VALUES(3, 1, 'd');
+  INSERT INTO t2 VALUES(3, 2, 'c');
+  INSERT INTO t2 VALUES(4, 1, 'b');
+  INSERT INTO t2 VALUES(4, 2, 'a');
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2;
+    SELECT c FROM t2 ORDER BY 1;
+  ROLLBACK;
+} {a c e f g h}
+
+do_execsql_test 2.2 {
+  BEGIN;
+    UPDATE t2 SET c=NULL ORDER BY a, b DESC LIMIT 3 OFFSET 1;
+    SELECT a, b, c FROM t2;
+  ROLLBACK;
+} {
+  1 1 {} 
+  1 2 g 
+  2 1 {} 
+  2 2 {} 
+  3 1 d 
+  3 2 c 
+  4 1 b 
+  4 2 a
+}
+
+
+
+finish_test
+

From aca84e6a8bb48f407b67994e0af93f7bf9099112 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 10 Nov 2017 12:41:21 +0000
Subject: [PATCH 265/270] Fix harmless compiler warning seen with MSVC.

FossilOrigin-Name: 3711ef2366af8fefccaaa0a6ee520ce6bc9c74a4fe0666f0a85ef96be46e02d3
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/dbpage.c  |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 9a2dbc0bc9..20d7f8ee03 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disallow\sORDER\sBY\sand\sLIMIT\son\sUPDATE\sand\sDELETE\sof\sviews\sand\sWITHOUT\sROWID\ntables.\s\sThis\sis\sa\stemporary\sfix\sfor\sticket\s[d4beea1633f1b88f]\suntil\sa\sbetter\nsolution\scan\sbe\sfound.
-D 2017-11-09T03:55:09.531
+C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC.
+D 2017-11-10T12:41:21.768
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -420,7 +420,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
-F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
+F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
@@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1bc2d04645c5239ba9a30a13f6fb3c8cc46461c864e7927012bc0ad305eb705e
-R a028f3be0864f739aeb3787daf46fc8e
-U drh
-Z 97937a65b1ac3d4b961579eac7d25cbd
+P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
+R 480986b4349d5fc99d8b880800c9dbc1
+U mistachkin
+Z 0f581c91a91b83b96a80c021b9d15d0e
diff --git a/manifest.uuid b/manifest.uuid
index e270b8e48e..476c0bbc49 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
\ No newline at end of file
+3711ef2366af8fefccaaa0a6ee520ce6bc9c74a4fe0666f0a85ef96be46e02d3
\ No newline at end of file
diff --git a/src/dbpage.c b/src/dbpage.c
index 75e4c42e67..c38de3b39f 100644
--- a/src/dbpage.c
+++ b/src/dbpage.c
@@ -303,7 +303,7 @@ static int dbpageUpdate(
   sqlite_int64 *pRowid
 ){
   DbpageTable *pTab = (DbpageTable *)pVtab;
-  int pgno;
+  Pgno pgno;
   DbPage *pDbPage = 0;
   int rc = SQLITE_OK;
   char *zErr = 0;
@@ -318,7 +318,7 @@ static int dbpageUpdate(
     goto update_fail;
   }
   pgno = sqlite3_value_int(argv[0]);
-  if( sqlite3_value_int(argv[1])!=pgno ){
+  if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){
     zErr = "cannot insert";
     goto update_fail;
   }

From 26caf5bef16bf48e2ea026104807b23f83c95dab Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 10 Nov 2017 15:42:21 +0000
Subject: [PATCH 266/270] Fix a problem with (DELETE...LIMIT) statements
 against WITHOUT ROWID tables with a single column PK.

FossilOrigin-Name: 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc
---
 manifest              | 19 ++++-----
 manifest.uuid         |  2 +-
 src/delete.c          |  4 +-
 src/parse.y           |  3 --
 test/wherelimit2.test | 89 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 101 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 4f63569478..72f23a851b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfor\sviews\sand\sWITHOUT\sROWID\stables.
-D 2017-11-09T19:53:06.815
+C Fix\sa\sproblem\swith\s(DELETE...LIMIT)\sstatements\sagainst\sWITHOUT\sROWID\stables\nwith\sa\ssingle\scolumn\sPK.
+D 2017-11-10T15:42:21.918
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c 001653cc4065ad83d47f4307c29572437f778f1fe249d482894e100275d27efd
+F src/delete.c cd86beec4f64491270f52521a45f0d335e29766b165511d025ee11119839375a
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
@@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 5fbc16a4adf5da8ee22a1ef271e001fd6b766eebc3195b52ba2c30cecbbaf2d0
+F src/parse.y 49eb13a590f88a03e81f1bc8df0634e56598ebd8d76d38dda24ba9edd7f92194
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -1558,7 +1558,7 @@ F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8
-F test/wherelimit2.test 501e470c9cf62e96359e3386ae520874792b83322060168ee9c708b2e953b985
+F test/wherelimit2.test 565fc74c3f96c3f08f3c20e233530e4c5532e6dfa01d1acc8dd2da5009a3dfd3
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
@@ -1674,10 +1674,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 62fe56b59270d9d7372b1bb8a53788a40d20d0f111fe38c61dd6269848592c70
-R 46792a4c59e5e3fb96fc57893e581fbe
-T *branch * update-delete-limit-fix
-T *sym-update-delete-limit-fix *
-T -sym-trunk *
+P 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce
+R d15b259ca771d410cbcab13660cb7826
 U dan
-Z c86d100b98964de2c810aa6a7710c91c
+Z 4ca2b1e389083d1c64078c78c67450a8
diff --git a/manifest.uuid b/manifest.uuid
index e7ae4dab5c..ebde4c85ea 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce
\ No newline at end of file
+35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index 509fd8af8d..34c5feb8b7 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -181,7 +181,9 @@ Expr *sqlite3LimitWhere(
   }else{
     Index *pPk = sqlite3PrimaryKeyIndex(pTab);
     if( pPk->nKeyCol==1 ){
-      pLhs = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[0]].zName);
+      const char *zName = pTab->aCol[pPk->aiColumn[0]].zName;
+      pLhs = sqlite3Expr(db, TK_ID, zName);
+      pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName));
     }else{
       int i;
       for(i=0; i<pPk->nKeyCol; i++){
diff --git a/src/parse.y b/src/parse.y
index 928e53dd82..2a536b0a8e 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -778,9 +778,6 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-#if 0
-  W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE");
-#endif
   sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset);
 }
 %endif
diff --git a/test/wherelimit2.test b/test/wherelimit2.test
index e505da42bb..1bf0168b50 100644
--- a/test/wherelimit2.test
+++ b/test/wherelimit2.test
@@ -22,6 +22,9 @@ ifcapable !update_delete_limit {
   return
 }
 
+#-------------------------------------------------------------------------
+# Test with views and INSTEAD OF triggers.
+#
 do_execsql_test 1.0 {
   CREATE TABLE t1(a, b);
   INSERT INTO t1 VALUES(1, 'f');
@@ -68,6 +71,9 @@ do_execsql_test 1.4 {
   update 6 update 5 update 4
 }
 
+#-------------------------------------------------------------------------
+# Simple test using WITHOUT ROWID table.
+#
 do_execsql_test 2.0 {
   CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
   INSERT INTO t2 VALUES(1, 1, 'h');
@@ -103,6 +109,89 @@ do_execsql_test 2.2 {
   4 2 a
 }
 
+#-------------------------------------------------------------------------
+# Test using a virtual table
+#
+ifcapable fts5 {
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(x);
+    INSERT INTO ft(rowid, x) VALUES(-45,   'a a');
+    INSERT INTO ft(rowid, x) VALUES(12,    'a b');
+    INSERT INTO ft(rowid, x) VALUES(444,   'a c');
+    INSERT INTO ft(rowid, x) VALUES(12300, 'a d');
+    INSERT INTO ft(rowid, x) VALUES(25400, 'a c');
+    INSERT INTO ft(rowid, x) VALUES(25401, 'a b');
+    INSERT INTO ft(rowid, x) VALUES(50000, 'a a');
+  }
+
+  do_execsql_test 3.1.1 {
+    BEGIN;
+      DELETE FROM ft ORDER BY rowid LIMIT 3;
+      SELECT x FROM ft;
+    ROLLBACK;
+  } {{a d} {a c} {a b} {a a}}
+
+  do_execsql_test 3.1.2 {
+    BEGIN;
+      DELETE FROM ft WHERE ft MATCH 'a' ORDER BY rowid LIMIT 3;
+      SELECT x FROM ft;
+    ROLLBACK;
+  } {{a d} {a c} {a b} {a a}}
+  
+  do_execsql_test 3.1.3 {
+    BEGIN;
+      DELETE FROM ft WHERE ft MATCH 'b' ORDER BY rowid ASC LIMIT 1 OFFSET 1;
+      SELECT rowid FROM ft;
+    ROLLBACK;
+  } {-45 12 444 12300 25400 50000}
+
+  do_execsql_test 3.2.1 {
+    BEGIN;
+      UPDATE ft SET x='hello' ORDER BY rowid LIMIT 2 OFFSET 2;
+      SELECT x FROM ft;
+    ROLLBACK;
+  } {{a a} {a b} hello hello {a c} {a b} {a a}}
+
+  do_execsql_test 3.2.2 {
+    BEGIN;
+      UPDATE ft SET x='hello' WHERE ft MATCH 'a' 
+          ORDER BY rowid DESC LIMIT 2 OFFSET 2;
+      SELECT x FROM ft;
+    ROLLBACK;
+  } {{a a} {a b} {a c} hello hello {a b} {a a}}
+} ;# fts5
+
+#-------------------------------------------------------------------------
+# Test using INDEXED BY clauses.
+#
+foreach {tn t} {1 "" 2 "WITHOUT ROWID"} {
+  execsql "DROP TABLE IF EXISTS x1"
+  execsql "CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d) $t"
+  do_execsql_test 4.$tn.0 {
+    CREATE INDEX x1bc ON x1(b, c);
+    INSERT INTO x1 VALUES(1,1,1,1);
+    INSERT INTO x1 VALUES(2,1,2,2);
+    INSERT INTO x1 VALUES(3,2,1,3);
+    INSERT INTO x1 VALUES(4,2,2,3);
+    INSERT INTO x1 VALUES(5,3,1,2);
+    INSERT INTO x1 VALUES(6,3,2,1);
+  }
+
+  do_execsql_test 4.$tn.1 {
+    BEGIN;
+      DELETE FROM x1 ORDER BY a LIMIT 2;
+      SELECT a FROM x1;
+    ROLLBACK;
+  } {3 4 5 6}
+}
+
+
+
+
+#-------------------------------------------------------------------------
+# Test using object names that require quoting.
+#
+
 
 
 finish_test

From ca3e3c3f1a4c2efdde91a36f35a2ce7df47c02f9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 10 Nov 2017 16:14:26 +0000
Subject: [PATCH 267/270] Fix a problem involving "DELETE/UPDATE...LIMIT"
 statements that use an INDEXED BY clause.

FossilOrigin-Name: 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb
---
 manifest              | 14 +++----
 manifest.uuid         |  2 +-
 src/delete.c          |  1 +
 test/wherelimit2.test | 92 ++++++++++++++++++++++++++++++++-----------
 4 files changed, 79 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index 72f23a851b..2d902bb312 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\s(DELETE...LIMIT)\sstatements\sagainst\sWITHOUT\sROWID\stables\nwith\sa\ssingle\scolumn\sPK.
-D 2017-11-10T15:42:21.918
+C Fix\sa\sproblem\sinvolving\s"DELETE/UPDATE...LIMIT"\sstatements\sthat\suse\san\sINDEXED\nBY\sclause.
+D 2017-11-10T16:14:26.225
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -422,7 +422,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c cd86beec4f64491270f52521a45f0d335e29766b165511d025ee11119839375a
+F src/delete.c 2b9f2d34c9420cbe6d83e99d08d735d8ba6b70729123b8c50909a7fa74a048d3
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
@@ -1558,7 +1558,7 @@ F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8
-F test/wherelimit2.test 565fc74c3f96c3f08f3c20e233530e4c5532e6dfa01d1acc8dd2da5009a3dfd3
+F test/wherelimit2.test 85e3fc3e61e100d8da27323c5cb30ed5881013c3b9100bb26ac5fd52d594c79b
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
@@ -1674,7 +1674,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 584b88aaf89ab30cb51185396b7b50c6ecba286add023a26ab41d865b9c605ce
-R d15b259ca771d410cbcab13660cb7826
+P 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc
+R 07fcc16cd40c06e7e9266df3e5decf1a
 U dan
-Z 4ca2b1e389083d1c64078c78c67450a8
+Z 02c3e7941017d253308184d6859dc937
diff --git a/manifest.uuid b/manifest.uuid
index ebde4c85ea..03929fbdcf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc
\ No newline at end of file
+09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index 34c5feb8b7..0d8737756a 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -202,6 +202,7 @@ Expr *sqlite3LimitWhere(
   pSrc->a[0].pTab = 0;
   pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0);
   pSrc->a[0].pTab = pTab;
+  pSrc->a[0].pIBIndex = 0;
 
   /* generate the SELECT expression tree. */
   pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, 
diff --git a/test/wherelimit2.test b/test/wherelimit2.test
index 1bf0168b50..fbcc14aa50 100644
--- a/test/wherelimit2.test
+++ b/test/wherelimit2.test
@@ -74,7 +74,7 @@ do_execsql_test 1.4 {
 #-------------------------------------------------------------------------
 # Simple test using WITHOUT ROWID table.
 #
-do_execsql_test 2.0 {
+do_execsql_test 2.1.0 {
   CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
   INSERT INTO t2 VALUES(1, 1, 'h');
   INSERT INTO t2 VALUES(1, 2, 'g');
@@ -86,14 +86,14 @@ do_execsql_test 2.0 {
   INSERT INTO t2 VALUES(4, 2, 'a');
 }
 
-do_execsql_test 2.1 {
+do_execsql_test 2.1.1 {
   BEGIN;
     DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2;
     SELECT c FROM t2 ORDER BY 1;
   ROLLBACK;
 } {a c e f g h}
 
-do_execsql_test 2.2 {
+do_execsql_test 2.1.2 {
   BEGIN;
     UPDATE t2 SET c=NULL ORDER BY a, b DESC LIMIT 3 OFFSET 1;
     SELECT a, b, c FROM t2;
@@ -109,6 +109,42 @@ do_execsql_test 2.2 {
   4 2 a
 }
 
+do_execsql_test 2.2.0 {
+  DROP TABLE t2;
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c) WITHOUT ROWID;
+  INSERT INTO t2 VALUES(1, 1, 'h');
+  INSERT INTO t2 VALUES(2, 2, 'g');
+  INSERT INTO t2 VALUES(3, 1, 'f');
+  INSERT INTO t2 VALUES(4, 2, 'e');
+  INSERT INTO t2 VALUES(5, 1, 'd');
+  INSERT INTO t2 VALUES(6, 2, 'c');
+  INSERT INTO t2 VALUES(7, 1, 'b');
+  INSERT INTO t2 VALUES(8, 2, 'a');
+}
+
+do_execsql_test 2.2.1 {
+  BEGIN;
+    DELETE FROM t2 WHERE b=1 ORDER BY c LIMIT 2;
+    SELECT c FROM t2 ORDER BY 1;
+  ROLLBACK;
+} {a c e f g h}
+
+do_execsql_test 2.2.2 {
+  BEGIN;
+    UPDATE t2 SET c=NULL ORDER BY a DESC LIMIT 3 OFFSET 1;
+    SELECT a, b, c FROM t2;
+  ROLLBACK;
+} {
+  1 1 h
+  2 2 g 
+  3 1 f
+  4 2 e
+  5 1 {}
+  6 2 {} 
+  7 1 {} 
+  8 2 a
+}
+
 #-------------------------------------------------------------------------
 # Test using a virtual table
 #
@@ -164,29 +200,41 @@ ifcapable fts5 {
 #-------------------------------------------------------------------------
 # Test using INDEXED BY clauses.
 #
-foreach {tn t} {1 "" 2 "WITHOUT ROWID"} {
-  execsql "DROP TABLE IF EXISTS x1"
-  execsql "CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d) $t"
-  do_execsql_test 4.$tn.0 {
-    CREATE INDEX x1bc ON x1(b, c);
-    INSERT INTO x1 VALUES(1,1,1,1);
-    INSERT INTO x1 VALUES(2,1,2,2);
-    INSERT INTO x1 VALUES(3,2,1,3);
-    INSERT INTO x1 VALUES(4,2,2,3);
-    INSERT INTO x1 VALUES(5,3,1,2);
-    INSERT INTO x1 VALUES(6,3,2,1);
-  }
-
-  do_execsql_test 4.$tn.1 {
-    BEGIN;
-      DELETE FROM x1 ORDER BY a LIMIT 2;
-      SELECT a FROM x1;
-    ROLLBACK;
-  } {3 4 5 6}
+do_execsql_test 4.0 {
+  CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c, d);
+  CREATE INDEX x1bc ON x1(b, c);
+  INSERT INTO x1 VALUES(1,1,1,1);
+  INSERT INTO x1 VALUES(2,1,2,2);
+  INSERT INTO x1 VALUES(3,2,1,3);
+  INSERT INTO x1 VALUES(4,2,2,3);
+  INSERT INTO x1 VALUES(5,3,1,2);
+  INSERT INTO x1 VALUES(6,3,2,1);
 }
 
+do_execsql_test 4.1 {
+  BEGIN;
+    DELETE FROM x1 ORDER BY a LIMIT 2;
+    SELECT a FROM x1;
+  ROLLBACK;
+} {3 4 5 6}
 
+do_catchsql_test 4.2 {
+  DELETE FROM x1 INDEXED BY x1bc WHERE d=3 LIMIT 1;
+} {1 {no query solution}}
 
+do_execsql_test 4.3 {
+  DELETE FROM x1 INDEXED BY x1bc WHERE b=3 LIMIT 1;
+  SELECT a FROM x1;
+} {1 2 3 4 6}
+
+do_catchsql_test 4.4 {
+  UPDATE x1 INDEXED BY x1bc SET d=5 WHERE d=3 LIMIT 1;
+} {1 {no query solution}}
+
+do_execsql_test 4.5 {
+  UPDATE x1 INDEXED BY x1bc SET d=5 WHERE b=2 LIMIT 1;
+  SELECT a, d FROM x1;
+} {1 1 2 2 3 5 4 3 6 1}
 
 #-------------------------------------------------------------------------
 # Test using object names that require quoting.

From dc32b448d31e9a22ffa190a60b2561c65d32aebf Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 10 Nov 2017 17:47:54 +0000
Subject: [PATCH 268/270] Add further tests for the code on this branch.

FossilOrigin-Name: f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669
---
 manifest              | 13 +++----
 manifest.uuid         |  2 +-
 test/wherelfault.test | 82 +++++++++++++++++++++++++++++++++++++++++++
 test/wherelimit2.test | 54 ++++++++++++++++++++++++++++
 4 files changed, 144 insertions(+), 7 deletions(-)
 create mode 100644 test/wherelfault.test

diff --git a/manifest b/manifest
index 2d902bb312..5bd57a17fd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sinvolving\s"DELETE/UPDATE...LIMIT"\sstatements\sthat\suse\san\sINDEXED\nBY\sclause.
-D 2017-11-10T16:14:26.225
+C Add\sfurther\stests\sfor\sthe\scode\son\sthis\sbranch.
+D 2017-11-10T17:47:54.147
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -1557,8 +1557,9 @@ F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
+F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3
 F test/wherelimit.test 1dee70c9cc147330156d75e23de88f771e624998b03ae316cb64e1d249f129d8
-F test/wherelimit2.test 85e3fc3e61e100d8da27323c5cb30ed5881013c3b9100bb26ac5fd52d594c79b
+F test/wherelimit2.test be78ba3aa1831c6358fd7d5b9809bfd520f0c2a7d63a295e8f182e140ff137c3
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
@@ -1674,7 +1675,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 35477a3dcceadf5dade8e036d5a2ce91b9ca83c4b85d309db233bdbcf538b1cc
-R 07fcc16cd40c06e7e9266df3e5decf1a
+P 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb
+R 276fb9c7ab900388b15c7da93d892ba2
 U dan
-Z 02c3e7941017d253308184d6859dc937
+Z 620b60e1c2be9813cc742e8964032235
diff --git a/manifest.uuid b/manifest.uuid
index 03929fbdcf..1df45c1eee 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb
\ No newline at end of file
+f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669
\ No newline at end of file
diff --git a/test/wherelfault.test b/test/wherelfault.test
new file mode 100644
index 0000000000..cfb8c1cfb4
--- /dev/null
+++ b/test/wherelfault.test
@@ -0,0 +1,82 @@
+# 2008 October 6
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing fault-injection with the 
+# LIMIT ... OFFSET ... clause of UPDATE and DELETE statements.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix wherelfault
+
+ifcapable !update_delete_limit {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'f');
+  INSERT INTO t1 VALUES(2, 'e');
+  INSERT INTO t1 VALUES(3, 'd');
+  INSERT INTO t1 VALUES(4, 'c');
+  INSERT INTO t1 VALUES(5, 'b');
+  INSERT INTO t1 VALUES(6, 'a');
+
+  CREATE VIEW v1 AS SELECT a,b FROM t1;
+  CREATE TABLE log(op, a);
+
+  CREATE TRIGGER v1del INSTEAD OF DELETE ON v1 BEGIN
+    INSERT INTO log VALUES('delete', old.a);
+  END;
+
+  CREATE TRIGGER v1upd INSTEAD OF UPDATE ON v1 BEGIN
+    INSERT INTO log VALUES('update', old.a);
+  END;
+}
+
+faultsim_save_and_close
+do_faultsim_test 1.1 -prep {
+  faultsim_restore_and_reopen
+  db eval {SELECT * FROM sqlite_master}
+} -body {
+  execsql { DELETE FROM v1 ORDER BY a LIMIT 3; }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+do_faultsim_test 1.2 -prep {
+  faultsim_restore_and_reopen
+  db eval {SELECT * FROM sqlite_master}
+} -body {
+  execsql { UPDATE v1 SET b = 555 ORDER BY a LIMIT 3 }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+#-------------------------------------------------------------------------
+sqlite3 db test.db
+do_execsql_test 2.1.0 {
+  CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
+}
+faultsim_save_and_close
+
+do_faultsim_test 2.1 -prep {
+  faultsim_restore_and_reopen
+  db eval {SELECT * FROM sqlite_master}
+} -body {
+  execsql { DELETE FROM t2 WHERE c=? ORDER BY a DESC LIMIT 10 }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+finish_test
diff --git a/test/wherelimit2.test b/test/wherelimit2.test
index fbcc14aa50..43a37958ee 100644
--- a/test/wherelimit2.test
+++ b/test/wherelimit2.test
@@ -239,7 +239,61 @@ do_execsql_test 4.5 {
 #-------------------------------------------------------------------------
 # Test using object names that require quoting.
 #
+do_execsql_test 5.0 {
+  CREATE TABLE "x y"("a b" PRIMARY KEY, "c d") WITHOUT ROWID;
+  CREATE INDEX xycd ON "x y"("c d");
 
+  INSERT INTO "x y" VALUES('a', 'a');
+  INSERT INTO "x y" VALUES('b', 'b');
+  INSERT INTO "x y" VALUES('c', 'c');
+  INSERT INTO "x y" VALUES('d', 'd');
+  INSERT INTO "x y" VALUES('e', 'a');
+  INSERT INTO "x y" VALUES('f', 'b');
+  INSERT INTO "x y" VALUES('g', 'c');
+  INSERT INTO "x y" VALUES('h', 'd');
+}
+
+do_execsql_test 5.1 {
+  BEGIN;
+    DELETE FROM "x y" WHERE "c d"!='e' ORDER BY "c d" LIMIT 2 OFFSET 2;
+    SELECT * FROM "x y" ORDER BY 1;
+  ROLLBACK;
+} {
+  a a c c d d e a g c h d
+}
+
+do_execsql_test 5.2 {
+  BEGIN;
+    UPDATE "x y" SET "c d"='e' WHERE "c d"!='e' ORDER BY "c d" LIMIT 2 OFFSET 2;
+    SELECT * FROM "x y" ORDER BY 1;
+  ROLLBACK;
+} {
+  a a b e c c d d e a f e g c h d
+}
+
+proc log {args} { lappend ::log {*}$args }
+db func log log
+do_execsql_test 5.3 {
+  CREATE VIEW "v w" AS SELECT * FROM "x y";
+  CREATE TRIGGER tr1 INSTEAD OF DELETE ON "v w" BEGIN
+    SELECT log(old."a b", old."c d");
+  END;
+  CREATE TRIGGER tr2 INSTEAD OF UPDATE ON "v w" BEGIN
+    SELECT log(new."a b", new."c d");
+  END;
+}
+
+do_test 5.4 {
+  set ::log {}
+  execsql { DELETE FROM "v w" ORDER BY "a b" LIMIT 3 }
+  set ::log
+} {a a b b c c}
+
+do_test 5.5 {
+  set ::log {}
+  execsql { UPDATE "v w" SET "a b" = "a b" || 'x' ORDER BY "a b" LIMIT 5; }
+  set ::log
+} {ax a bx b cx c dx d ex a}
 
 
 finish_test

From 3b61ebb8de161ffdc505d83d9acd9b881bb139bf Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 10 Nov 2017 20:13:14 +0000
Subject: [PATCH 269/270] Omit some extra code from
 non-SQLITE_ENABLE_UPDATE_DELETE_LIMIT builds.

FossilOrigin-Name: 72be33f9c84de3ec4afc40549482417456ca82c1d16b473dc034b144055271e5
---
 manifest        | 24 ++++++++++++------------
 manifest.uuid   |  2 +-
 src/delete.c    | 10 ++++++----
 src/fkey.c      |  2 +-
 src/parse.y     |  8 ++++----
 src/resolve.c   |  5 +----
 src/sqliteInt.h |  7 ++-----
 src/trigger.c   |  4 ++--
 src/update.c    | 10 ++++++----
 9 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/manifest b/manifest
index 5bd57a17fd..e5b0bb5863 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfurther\stests\sfor\sthe\scode\son\sthis\sbranch.
-D 2017-11-10T17:47:54.147
+C Omit\ssome\sextra\scode\sfrom\snon-SQLITE_ENABLE_UPDATE_DELETE_LIMIT\sbuilds.
+D 2017-11-10T20:13:14.097
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -422,10 +422,10 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c 2b9f2d34c9420cbe6d83e99d08d735d8ba6b70729123b8c50909a7fa74a048d3
+F src/delete.c f97cc2a9a30b4119ec5ed0da4e41335060cbb8510a5f1a756be48c065cb824f0
 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
-F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333
+F src/fkey.c 821f70b6c43ba4542ffb5fdf11d3ac5f79e4cbae4750ee3de379afbc5e1f7781
 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
@@ -459,7 +459,7 @@ F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 49eb13a590f88a03e81f1bc8df0634e56598ebd8d76d38dda24ba9edd7f92194
+F src/parse.y f5f02ef39444982af36545bd52ae2921fc4ba1bac24d9b11efd8ec1f52c5b4dc
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -468,14 +468,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
 F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 704978e5aabb9f524789e3b53016b4068a4e64c669c0f8ff025d2b23b95d62b5
+F src/resolve.c 5b1e89ba279f4a4ab2f0975a7100d75be71e1a43a2df75a9c909d45bdd18c6ed
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
 F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 821479be791acfdb63177b21a491e604526191c1630d1d65d7a4a7c65d203c83
+F src/sqliteInt.h abd4e64bc72906449d801d0e211265525239bc021bd9b7a72143c281fc24fa03
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -534,8 +534,8 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef
-F src/trigger.c 48e0f7ed6749ce4d50a695e09e20ce9cf84ecabf2691852c965a51e0b620eccc
-F src/update.c 5123fcb4aa98a705ced2acb172c1761a570d142d77901b4f4ce38acb191ef8ed
+F src/trigger.c 929b12bb5f9ab3b10c589f71fa176effe6ea8bddc9eb1bbd2439755fa0acdfdf
+F src/update.c 2e7ef3839eb620e9140a41bbfd05e29d7d216441c5d97221b75635dea4e49672
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
@@ -1675,7 +1675,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 09f94c2c8199b0d23a45cc062ca9561f9e5ddfcba117100e41889ce199d21bdb
-R 276fb9c7ab900388b15c7da93d892ba2
+P f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669
+R 1176f1b5b2e02ce4a3ab41b0af2d65f4
 U dan
-Z 620b60e1c2be9813cc742e8964032235
+Z f8079371239b2151fe43a05b53b9cf38
diff --git a/manifest.uuid b/manifest.uuid
index 1df45c1eee..4ef7d391b2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f8c4e33f4813e0c909064406b5cc17e2d465d8a48a50ede1d356b39479d3d669
\ No newline at end of file
+72be33f9c84de3ec4afc40549482417456ca82c1d16b473dc034b144055271e5
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index 0d8737756a..5c96fdfb8d 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -224,13 +224,13 @@ Expr *sqlite3LimitWhere(
 **                 \________/       \________________/
 **                  pTabList              pWhere
 */
-void sqlite3DeleteFromLimit(
+void sqlite3DeleteFrom(
   Parse *pParse,         /* The parser context */
   SrcList *pTabList,     /* The table from which we should delete things */
   Expr *pWhere,          /* The WHERE clause.  May be null */
-  ExprList *pOrderBy,
-  Expr *pLimit,
-  Expr *pOffset
+  ExprList *pOrderBy,    /* ORDER BY clause. May be null */
+  Expr *pLimit,          /* LIMIT clause. May be null */
+  Expr *pOffset          /* OFFSET clause. May be null */
 ){
   Vdbe *v;               /* The virtual database engine */
   Table *pTab;           /* The table from which records will be deleted */
@@ -606,9 +606,11 @@ delete_from_cleanup:
   sqlite3AuthContextPop(&sContext);
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprDelete(db, pWhere);
+#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) 
   sqlite3ExprListDelete(db, pOrderBy);
   sqlite3ExprDelete(db, pLimit);
   sqlite3ExprDelete(db, pOffset);
+#endif
   sqlite3DbFree(db, aToOpen);
   return;
 }
diff --git a/src/fkey.c b/src/fkey.c
index 0012768a39..44ef3c7c7b 100644
--- a/src/fkey.c
+++ b/src/fkey.c
@@ -725,7 +725,7 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){
     }
 
     pParse->disableTriggers = 1;
-    sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0);
+    sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0, 0);
     pParse->disableTriggers = 0;
 
     /* If the DELETE has generated immediate foreign key constraint 
diff --git a/src/parse.y b/src/parse.y
index 2a536b0a8e..18f587c05d 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -753,14 +753,14 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W)
         orderby_opt(O) limit_opt(L). {
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
-  sqlite3DeleteFromLimit(pParse,X,W,O,L.pLimit,L.pOffset); 
+  sqlite3DeleteFrom(pParse,X,W,O,L.pLimit,L.pOffset); 
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
 cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
-  sqlite3DeleteFrom(pParse,X,W);
+  sqlite3DeleteFrom(pParse,X,W,0,0,0);
 }
 %endif
 
@@ -778,7 +778,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3UpdateLimit(pParse,X,Y,W,R,O,L.pLimit,L.pOffset);
+  sqlite3Update(pParse,X,Y,W,R,O,L.pLimit,L.pOffset);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
@@ -787,7 +787,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3Update(pParse,X,Y,W,R);
+  sqlite3Update(pParse,X,Y,W,R,0,0,0);
 }
 %endif
 
diff --git a/src/resolve.c b/src/resolve.c
index 945654ead0..17dbbccfc5 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -597,10 +597,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       struct SrcList_item *pItem;
       assert( pSrcList && pSrcList->nSrc==1 );
       pItem = pSrcList->a;
-      if( !HasRowid(pItem->pTab) || pItem->pTab->pSelect!=0 ){
-         sqlite3ErrorMsg(pParse, "ORDER BY and LIMIT not support for table %s",
-                                 pItem->pTab->zName);
-      }
+      assert( HasRowid(pItem->pTab) && pItem->pTab->pSelect==0 );
       pExpr->op = TK_COLUMN;
       pExpr->pTab = pItem->pTab;
       pExpr->iTable = pItem->iCursor;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index d3c412e222..95a66b4fe7 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3764,11 +3764,8 @@ void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
 Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*);
 #endif
-void sqlite3DeleteFromLimit(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*);
-#define sqlite3DeleteFrom(x,y,z) sqlite3DeleteFromLimit(x,y,z,0,0,0)
-void sqlite3UpdateLimit(Parse*, SrcList*, ExprList*, Expr*, int,
-                        ExprList*,Expr*,Expr*);
-#define sqlite3Update(v,w,x,y,z) sqlite3UpdateLimit(v,w,x,y,z,0,0,0)
+void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*);
+void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,Expr*);
 WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
 void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
diff --git a/src/trigger.c b/src/trigger.c
index 2a4fdd8ed4..a17769ae98 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -711,7 +711,7 @@ static int codeTriggerProgram(
           targetSrcList(pParse, pStep),
           sqlite3ExprListDup(db, pStep->pExprList, 0), 
           sqlite3ExprDup(db, pStep->pWhere, 0), 
-          pParse->eOrconf
+          pParse->eOrconf, 0, 0, 0
         );
         break;
       }
@@ -727,7 +727,7 @@ static int codeTriggerProgram(
       case TK_DELETE: {
         sqlite3DeleteFrom(pParse, 
           targetSrcList(pParse, pStep),
-          sqlite3ExprDup(db, pStep->pWhere, 0)
+          sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0, 0
         );
         break;
       }
diff --git a/src/update.c b/src/update.c
index f63a9ffe8a..4cfa4019eb 100644
--- a/src/update.c
+++ b/src/update.c
@@ -86,15 +86,15 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
 **          \_______/ \________/     \______/       \________________/
 *            onError   pTabList      pChanges             pWhere
 */
-void sqlite3UpdateLimit(
+void sqlite3Update(
   Parse *pParse,         /* The parser context */
   SrcList *pTabList,     /* The table in which we should change things */
   ExprList *pChanges,    /* Things to be changed */
   Expr *pWhere,          /* The WHERE clause.  May be null */
   int onError,           /* How to handle constraint errors */
-  ExprList *pOrderBy,
-  Expr *pLimit,
-  Expr *pOffset
+  ExprList *pOrderBy,    /* ORDER BY clause. May be null */
+  Expr *pLimit,          /* LIMIT clause. May be null */
+  Expr *pOffset          /* OFFSET clause. May be null */
 ){
   int i, j;              /* Loop counters */
   Table *pTab;           /* The table to be updated */
@@ -745,9 +745,11 @@ update_cleanup:
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprListDelete(db, pChanges);
   sqlite3ExprDelete(db, pWhere);
+#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) 
   sqlite3ExprListDelete(db, pOrderBy);
   sqlite3ExprDelete(db, pLimit);
   sqlite3ExprDelete(db, pOffset);
+#endif
   return;
 }
 /* Make sure "isView" and other macros defined above are undefined. Otherwise

From cb0267185c8178ad94b9dc169a2ee043ec60321e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 11 Dec 2017 14:02:10 +0000
Subject: [PATCH 270/270] Abort on an invalid paramater to sqlite3BitvecSet().

FossilOrigin-Name: 163c870950f386f6b0bb1ff9b3886cf95ba0deed414cae75baf87621ed3528c2
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/bitvec.c  |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 77220b7e11..dce0a2ddfa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\schanges\sfrom\strunk.\sThis\sfixes\sthe\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfunctionality\sso\sthat\sit\sworks\swith\sviews\sand\sWITHOUT\sROWID\stables.
-D 2017-11-14T20:06:15.563
+C Abort\son\san\sinvalid\sparamater\sto\ssqlite3BitvecSet().
+D 2017-12-11T14:02:10.998
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -414,7 +414,7 @@ F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421
 F src/attach.c 07b706e336fd3cedbd855e1f8266d10e82fecae07daf86717b5760cd7784c584
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
-F src/bitvec.c c77b7f5759e413c1c8b53267d633c952e66db79c1171964c7e24c0f92f5019cf
+F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c a41d580524a4cd9b00443b7196d298e05463f6bfcb712853db3abcddc93cf3ab
 F src/btree.h feafd0647331366f4ef17f7e68597e9029f001e7ab16a125e2f176c598a7ef4a
@@ -1686,7 +1686,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7f217edab4575554f657d38e2a1bc6b3f577998fdbecb04eb200aeb8b8406581 dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a
-R d370973eeca86ae526a0060a468b38d6
-U dan
-Z 20f27c4758e319217032ba89b2abf6bd
+P d90e5f346bcf7adab26ca8dad9dfbd0fbb86604a15f2fe827f11b3faab036750
+R e43dfcde6e63fd81287681f03e602a2f
+U drh
+Z 3fc04e05119313efe5c6f01b2d450caa
diff --git a/manifest.uuid b/manifest.uuid
index b87d30350a..af787c32f4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d90e5f346bcf7adab26ca8dad9dfbd0fbb86604a15f2fe827f11b3faab036750
\ No newline at end of file
+163c870950f386f6b0bb1ff9b3886cf95ba0deed414cae75baf87621ed3528c2
\ No newline at end of file
diff --git a/src/bitvec.c b/src/bitvec.c
index 31994f550b..68ff12dc91 100644
--- a/src/bitvec.c
+++ b/src/bitvec.c
@@ -175,6 +175,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){
     sqlite3_log(SQLITE_ERROR, 
         "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize
     );
+    abort();
   }
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {